From 0bf7e0f688ce48a31c41be8d2751d2426ba5f0d1 Mon Sep 17 00:00:00 2001 From: asonix Date: Wed, 9 Dec 2020 23:05:04 -0600 Subject: [PATCH] Add initial details implementation Details is an object generated for each image, containing it's created_at date to properly set response headers. It also can be expanding in the future to include an image's dimensions, which we can expose via an API endpoint. In addition to this, Details allows us the ability to re-generate processed images. For now, all images which do not yet have an associated Details will be re-generated upon access. Details is set for each thumbnail on generation, so this will only happen once for each image. --- Cargo.lock | 136 +++++++-------------------- Cargo.toml | 4 +- README.md | 7 +- src/error.rs | 6 +- src/main.rs | 71 ++++++++------ src/migrate/mod.rs | 211 +----------------------------------------- src/migrate/s032.rs | 85 ----------------- src/processor.rs | 58 ++++++++++++ src/upload_manager.rs | 86 ++++++++++++++++- 9 files changed, 224 insertions(+), 440 deletions(-) delete mode 100644 src/migrate/s032.rs diff --git a/Cargo.lock b/Cargo.lock index 1f5165c..e7a6f2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -361,15 +361,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" - -[[package]] -name = "array-init" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30bbe2f5e3d117f55bd8c7a1f9191e4a5deba9f15f595bbea4f670c59c765db" +checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4" [[package]] name = "async-trait" @@ -564,9 +558,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cexpr" @@ -628,20 +622,11 @@ dependencies = [ "vec_map", ] -[[package]] -name = "cloudabi" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" -dependencies = [ - "bitflags", -] - [[package]] name = "const_fn" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" [[package]] name = "cookie" @@ -675,21 +660,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", -] - [[package]] name = "crossbeam-epoch" version = "0.9.1" @@ -698,23 +668,12 @@ checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" dependencies = [ "cfg-if 1.0.0", "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils", "lazy_static", - "memoffset 0.6.1", + "memoffset", "scopeguard", ] -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", -] - [[package]] name = "crossbeam-utils" version = "0.8.1" @@ -1208,9 +1167,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" [[package]] name = "libloading" @@ -1299,27 +1258,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.1" @@ -1389,9 +1333,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.36" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" dependencies = [ "cfg-if 0.1.10", "libc", @@ -1479,12 +1423,11 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" dependencies = [ - "cfg-if 0.1.10", - "cloudabi", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -1506,7 +1449,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pict-rs" -version = "0.2.6" +version = "0.3.0-alpha.0" dependencies = [ "actix-form-data", "actix-fs", @@ -1526,10 +1469,10 @@ dependencies = [ "serde", "serde_json", "sha2", - "sled 0.32.1", - "sled 0.34.6", + "sled", "structopt", "thiserror", + "time 0.2.23", "tracing", "tracing-futures", "tracing-subscriber", @@ -1856,18 +1799,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" dependencies = [ "proc-macro2", "quote", @@ -1960,24 +1903,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" -[[package]] -name = "sled" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3dbbb8ee10611bd1d020767c27599ccbbf8365f7e0ed7e54429cc8b9433ad8" -dependencies = [ - "array-init", - "backtrace", - "crc32fast", - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot", -] - [[package]] name = "sled" version = "0.34.6" @@ -1985,8 +1910,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" dependencies = [ "crc32fast", - "crossbeam-epoch 0.9.1", - "crossbeam-utils 0.8.1", + "crossbeam-epoch", + "crossbeam-utils", "fs2", "fxhash", "libc", @@ -1996,9 +1921,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" [[package]] name = "socket2" @@ -2108,9 +2033,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.53" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68" +checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" dependencies = [ "proc-macro2", "quote", @@ -2192,6 +2117,7 @@ checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" dependencies = [ "const_fn", "libc", + "serde", "standback", "stdweb", "time-macros", @@ -2239,9 +2165,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" dependencies = [ "bytes", "fnv", diff --git a/Cargo.toml b/Cargo.toml index bb53237..b3a0fca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pict-rs" description = "A simple image hosting service" -version = "0.2.6" +version = "0.3.0-alpha.0" authors = ["asonix "] license = "AGPL-3.0" readme = "README.md" @@ -27,10 +27,10 @@ rexiv2 = "0.9.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.9.0" -sled032 = { version = "0.32.0", package = "sled" } sled = { version = "0.34.4" } structopt = "0.3.14" thiserror = "1.0" +time = { version = "0.2.23", features = ["serde"] } tracing = "0.1.15" tracing-futures = "0.2.4" tracing-subscriber = { version = "0.2.5", features = ["fmt", "tracing-log"] } diff --git a/README.md b/README.md index ab9f783..ade539e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ _a simple image hosting service_ ## Usage ### Running ``` -pict-rs 0.2.0-alpha.3 +pict-rs 0.3.0-alpha.0 USAGE: pict-rs [FLAGS] [OPTIONS] --path @@ -109,7 +109,10 @@ pict-rs offers the following endpoints: existing transformations include - `identity=true`: apply no changes - `blur={float}`: apply a gaussian blur to the file - - `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square + - `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` + square using raw pixel sampling + - `resize={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square + using a Lanczos2 filter. This is slower than sampling but looks a bit better in some cases Supported `ext` file extensions include `png`, `jpg`, and `webp` diff --git a/src/error.rs b/src/error.rs index c8fafb4..3f94780 100644 --- a/src/error.rs +++ b/src/error.rs @@ -9,9 +9,6 @@ pub(crate) enum UploadError { #[error("Couldn't save file, {0}")] Save(#[from] actix_fs::Error), - #[error("Error in DB migration, {0}")] - Migrate(#[from] sled032::Error), - #[error("Error in DB, {0}")] Db(#[from] sled::Error), @@ -71,6 +68,9 @@ pub(crate) enum UploadError { #[error("Error in MagickWand, {0}")] Wand(String), + + #[error("{0}")] + Json(#[from] serde_json::Error), } impl From for UploadError { diff --git a/src/main.rs b/src/main.rs index eff8553..5fbf2dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,7 +26,7 @@ use self::{ error::UploadError, middleware::{Internal, Tracing}, processor::process_image, - upload_manager::UploadManager, + upload_manager::{Details, UploadManager}, validate::{image_webp, video_mp4}, }; @@ -289,38 +289,28 @@ async fn process( .map_err(|_| UploadError::UnsupportedFormat)?; let processed_name = format!("{}.{}", name, ext); let base = manager.image_dir(); - let mut path = self::processor::build_path(base, &chain, processed_name); - - if let Some((updated_path, exists)) = self::processor::prepare_image(path.clone()).await? { - path = updated_path.clone(); - - if exists.is_new() { - // Save the transcoded file in another task - debug!("Spawning storage task"); - let span = Span::current(); - let manager2 = manager.clone(); - let name = name.clone(); - actix_rt::spawn(async move { - let entered = span.enter(); - if let Err(e) = manager2.store_variant(updated_path, name).await { - error!("Error storing variant, {}", e); - return; - } - drop(entered); - }); - } - } + let thumbnail_path = self::processor::build_path(base, &chain, processed_name); // If the thumbnail doesn't exist, we need to create it - if let Err(e) = actix_fs::metadata(path.clone()).await { + let thumbnail_exists = if let Err(e) = actix_fs::metadata(thumbnail_path.clone()).await { if e.kind() != Some(std::io::ErrorKind::NotFound) { error!("Error looking up processed image, {}", e); return Err(e.into()); } + false + } else { + true + }; + let details = manager + .variant_details(thumbnail_path.clone(), name.clone()) + .await?; + + if !thumbnail_exists || details.is_none() { let mut original_path = manager.image_dir(); original_path.push(name.clone()); + // Create and save a JPG for motion images (gif, mp4) if let Some((updated_path, exists)) = self::processor::prepare_image(original_path.clone()).await? { @@ -346,14 +336,25 @@ async fn process( // apply chain to the provided image let img_bytes = process_image(original_path.clone(), chain, format).await?; - let path2 = path.clone(); + let path2 = thumbnail_path.clone(); let img_bytes2 = img_bytes.clone(); // Save the file in another task, we want to return the thumbnail now debug!("Spawning storage task"); let span = Span::current(); + let store_details = details.is_none(); actix_rt::spawn(async move { let entered = span.enter(); + if store_details { + debug!("Storing details"); + if let Err(e) = manager + .store_variant_details(path2.clone(), name.clone()) + .await + { + error!("Error storing details, {}", e); + return; + } + } if let Err(e) = manager.store_variant(path2.clone(), name).await { error!("Error storing variant, {}", e); return; @@ -365,23 +366,27 @@ async fn process( drop(entered); }); + let details = details.unwrap_or(Details::now()); + return Ok(srv_response( Box::pin(futures::stream::once(async { Ok(img_bytes) as Result<_, UploadError> })), content_type, 7 * DAYS, - SystemTime::now(), + details.system_time(), )); } - let stream = actix_fs::read_to_stream(path).await?; + let stream = actix_fs::read_to_stream(thumbnail_path).await?; + + let details = details.unwrap_or(Details::now()); Ok(srv_response( stream, content_type, 7 * DAYS, - SystemTime::now(), + details.system_time(), )) } @@ -394,7 +399,15 @@ async fn serve( let name = manager.from_alias(alias.into_inner()).await?; let content_type = from_name(&name)?; let mut path = manager.image_dir(); - path.push(name); + path.push(name.clone()); + + let details = manager.variant_details(path.clone(), name.clone()).await?; + + if details.is_none() { + manager.store_variant_details(path.clone(), name).await?; + } + + let details = details.unwrap_or(Details::now()); let stream = actix_fs::read_to_stream(path).await?; @@ -402,7 +415,7 @@ async fn serve( stream, content_type, 7 * DAYS, - SystemTime::now(), + details.system_time(), )) } diff --git a/src/migrate/mod.rs b/src/migrate/mod.rs index d82fb7a..99d1108 100644 --- a/src/migrate/mod.rs +++ b/src/migrate/mod.rs @@ -1,9 +1,7 @@ use crate::UploadError; use sled; use std::path::PathBuf; -use tracing::{debug, info, warn}; -mod s032; mod s034; type SledIter = Box, Vec), UploadError>>>; @@ -68,9 +66,7 @@ impl LatestDb { loop { let root_dir2 = root_dir.clone(); - let res = std::panic::catch_unwind(move || { - version.migrate(root_dir2) - }); + let res = std::panic::catch_unwind(move || version.migrate(root_dir2)); if let Ok(res) = res { return res; @@ -81,8 +77,6 @@ impl LatestDb { #[derive(Clone, Copy)] enum DbVersion { - Sled0320Rc1, - Sled032, Sled034, Fresh, } @@ -93,170 +87,16 @@ impl DbVersion { return DbVersion::Sled034; } - if s032::exists(root.clone()) { - return DbVersion::Sled032; - } - - if s032::exists_rc1(root.clone()) { - return DbVersion::Sled0320Rc1; - } - DbVersion::Fresh } fn migrate(self, root: PathBuf) -> Result { match self { - DbVersion::Sled0320Rc1 => migrate_0_32_0_rc1(root), - DbVersion::Sled032 => migrate_0_32(root), DbVersion::Sled034 | DbVersion::Fresh => s034::open(root), } } } -fn migrate_0_32_0_rc1(root: PathBuf) -> Result { - info!("Migrating database from 0.32.0-rc1 to 0.34.0"); - - let old_db = s032::open_rc1(root.clone())?; - let new_db = s034::open(root)?; - - migrate(old_db, new_db) -} - -fn migrate_0_32(root: PathBuf) -> Result { - info!("Migrating database from 0.32 to 0.34"); - - let old_db = s032::open(root.clone())?; - let new_db = s034::open(root)?; - - migrate(old_db, new_db) -} - -fn migrate(old_db: Old, new_db: New) -> Result -where - Old: SledDb, - New: SledDb, -{ - let old_alias_tree = old_db.open_tree("alias")?; - let new_alias_tree = new_db.open_tree("alias")?; - - let new_migrate_tree = new_db.open_tree("migrate")?; - if let Some(_) = new_migrate_tree.get("done")? { - return Ok(new_db); - } - - let (iterator, mut counter) = if let Some(last_migrated) = new_migrate_tree.get("last_migrated")? { - let mut last_migrated = String::from_utf8_lossy(&last_migrated).to_string(); - info!("Previous migration failed after {}, attempting to skip", last_migrated); - if let Some(index) = last_migrated.find('.') { - last_migrated = last_migrated.split_at(index).0.to_owned(); - } - if last_migrated.len() > 3 { - last_migrated = last_migrated.split_at(3).0.to_owned(); - } - let last_migrated = increment_alphanumeric(&last_migrated).as_bytes().to_owned(); - new_migrate_tree.insert("last_migrated", last_migrated.clone())?; - new_migrate_tree.flush()?; - if let Some(count) = new_migrate_tree.get("counter")? { - (old_alias_tree.range(last_migrated..), String::from_utf8_lossy(&count).parse::().unwrap()) - } else { - (old_alias_tree.range(last_migrated..), 0) - } - } else { - (old_alias_tree.iter(), 0) - }; - - for res in iterator { - let (k, _) = res?; - - if let Some(v) = old_alias_tree.get(&k)? { - if !k.contains(&b"/"[0]) { - if let Some(id) = old_alias_tree.get(alias_id_key(&String::from_utf8_lossy(&k)))? { - counter += 1; - debug!("Migrating alias #{}", counter); - // k is an alias - migrate_main_tree(&k, &v, &old_db, &new_db, &String::from_utf8_lossy(&id))?; - debug!( - "Moving alias -> hash for alias {}", - String::from_utf8_lossy(k.as_ref()), - ); - new_migrate_tree.insert("counter", format!("{}", counter))?; - } - } else { - debug!( - "Moving {}, {}", - String::from_utf8_lossy(k.as_ref()), - String::from_utf8_lossy(v.as_ref()) - ); - } - new_alias_tree.insert(k.clone(), v)?; - new_migrate_tree.insert("last_migrated", k)?; - new_alias_tree.flush()?; - new_migrate_tree.flush()?; - } else { - warn!("MISSING {}", String::from_utf8_lossy(k.as_ref())); - } - } - info!("Moved {} unique aliases", counter); - - new_migrate_tree.insert("done", "true")?; - new_migrate_tree.flush()?; - - Ok(new_db) -} - -fn migrate_main_tree( - alias: &[u8], - hash: &[u8], - old_db: Old, - new_db: New, - id: &str, -) -> Result<(), UploadError> -where - Old: SledDb, - New: SledDb, -{ - let main_tree = new_db.open_tree("main")?; - - let new_fname_tree = new_db.open_tree("filename")?; - - debug!( - "Migrating files for {}", - String::from_utf8_lossy(alias.as_ref()) - ); - if let Some(filename) = old_db.self_tree().get(&hash)? { - main_tree.insert(&hash, filename.clone())?; - new_fname_tree.insert(filename, hash.clone())?; - main_tree.flush()?; - new_fname_tree.flush()?; - } else { - warn!("Missing filename"); - } - - let key = alias_key(&hash, id); - if let Some(v) = old_db.self_tree().get(&key)? { - main_tree.insert(key, v)?; - main_tree.flush()?; - } else { - warn!("Not migrating alias {} id {}", String::from_utf8_lossy(&alias), id); - return Ok(()); - } - - let (start, end) = variant_key_bounds(&hash); - if main_tree.range(start.clone()..end.clone()).next().is_none() { - let mut counter = 0; - for res in old_db.self_tree().range(start.clone()..end.clone()) { - counter += 1; - let (k, v) = res?; - debug!("Moving variant #{} for {}", counter, String::from_utf8_lossy(v.as_ref())); - main_tree.insert(k, v)?; - main_tree.flush()?; - } - debug!("Moved {} variants for {}", counter, String::from_utf8_lossy(alias.as_ref())); - } - - Ok(()) -} - pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec, Vec) { let mut start = hash.to_vec(); start.extend(&[0]); @@ -289,52 +129,3 @@ pub(crate) fn alias_key(hash: &[u8], id: &str) -> Vec { key } - -const VALID: &[char] = &[ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', -]; - -fn increment_alphanumeric(input: &str) -> String { - let (_, output) = input.chars().rev().fold((true, String::new()), |(incr_next, mut acc), item| { - if incr_next { - let mut index = None; - for (i, test) in VALID.iter().enumerate() { - if *test == item { - index = Some(i); - } - } - let index = index.unwrap_or(0); - let (set_incr_next, next_index) = if index == (VALID.len() - 1) { - (true, 0) - } else { - (false, index + 1) - }; - acc.extend(&[VALID[next_index]]); - (set_incr_next, acc) - } else { - acc.extend(&[item]); - (false, acc) - } - }); - - output.chars().rev().collect() -} - -#[cfg(test)] -mod tests { - use super::increment_alphanumeric; - - #[test] - fn increments() { - assert_eq!(increment_alphanumeric("hello"), "hellp"); - assert_eq!(increment_alphanumeric("0"), "1"); - assert_eq!(increment_alphanumeric("9"), "A"); - assert_eq!(increment_alphanumeric("Z"), "a"); - assert_eq!(increment_alphanumeric("z"), "0"); - assert_eq!(increment_alphanumeric("az"), "b0"); - assert_eq!(increment_alphanumeric("19"), "1A"); - assert_eq!(increment_alphanumeric("AZ"), "Aa"); - } -} diff --git a/src/migrate/s032.rs b/src/migrate/s032.rs deleted file mode 100644 index 732b9dc..0000000 --- a/src/migrate/s032.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::{ - migrate::{SledDb, SledIter, SledTree}, - UploadError, -}; -use std::path::PathBuf; - -const SLED_032: &str = "db-0.32"; -const SLED_0320_RC1: &str = "db"; - -pub(crate) fn exists_rc1(mut base: PathBuf) -> bool { - base.push(SLED_0320_RC1); - - std::fs::metadata(base).is_ok() -} - -pub(crate) fn open_rc1(mut base: PathBuf) -> Result { - base.push(SLED_0320_RC1); - - Ok(sled032::open(base)?) -} - -pub(crate) fn exists(mut base: PathBuf) -> bool { - base.push("sled"); - base.push(SLED_032); - - std::fs::metadata(base).is_ok() -} - -pub(crate) fn open(mut base: PathBuf) -> Result { - base.push("sled"); - base.push(SLED_032); - - Ok(sled032::open(base)?) -} - -impl SledDb for sled032::Db { - type SledTree = sled032::Tree; - - fn open_tree(&self, name: &str) -> Result { - Ok(sled032::Db::open_tree(self, name)?) - } - - fn self_tree(&self) -> &Self::SledTree { - &*self - } -} - -impl SledTree for sled032::Tree { - fn get(&self, key: K) -> Result>, UploadError> - where - K: AsRef<[u8]>, - { - Ok(sled032::Tree::get(self, key)?.map(|v| Vec::from(v.as_ref()))) - } - - fn insert(&self, key: K, value: V) -> Result<(), UploadError> - where - K: AsRef<[u8]>, - V: AsRef<[u8]>, - { - Ok(sled032::Tree::insert(self, key, value.as_ref().to_vec()).map(|_| ())?) - } - - fn iter(&self) -> SledIter { - Box::new(sled032::Tree::iter(self).map(|res| { - res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec())) - .map_err(UploadError::from) - })) - } - - fn range(&self, range: R) -> SledIter - where - K: AsRef<[u8]>, - R: std::ops::RangeBounds, - { - Box::new(sled032::Tree::range(self, range).map(|res| { - res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec())) - .map_err(UploadError::from) - })) - } - - fn flush(&self) -> Result<(), UploadError> { - sled032::Tree::flush(self).map(|_| ()).map_err(UploadError::from) - } -} diff --git a/src/processor.rs b/src/processor.rs index 3c49021..72fee67 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -113,6 +113,63 @@ impl Processor for Thumbnail { } } +pub(crate) struct Resize(usize); + +impl Processor for Resize { + fn name() -> &'static str + where + Self: Sized, + { + "resize" + } + + fn is_processor(s: &str) -> bool + where + Self: Sized, + { + s == Self::name() + } + + fn parse(_: &str, v: &str) -> Option> + where + Self: Sized, + { + let size = v.parse().ok()?; + Some(Box::new(Resize(size))) + } + + fn path(&self, mut path: PathBuf) -> PathBuf { + path.push(Self::name()); + path.push(self.0.to_string()); + path + } + + fn process(&self, wand: &mut MagickWand) -> Result<(), UploadError> { + debug!("Resize"); + let width = wand.get_image_width(); + let height = wand.get_image_height(); + + if width > self.0 || height > self.0 { + let width_ratio = width as f64 / self.0 as f64; + let height_ratio = height as f64 / self.0 as f64; + + let (new_width, new_height) = if width_ratio < height_ratio { + (width as f64 / height_ratio, self.0 as f64) + } else { + (self.0 as f64, height as f64 / width_ratio) + }; + + wand.resize_image( + new_width as usize, + new_height as usize, + magick_rust::bindings::FilterType_Lanczos2Filter, + ); + } + + Ok(()) + } +} + pub(crate) struct Blur(f64); impl Processor for Blur { @@ -178,6 +235,7 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> ProcessChain { parse!(Identity, k, v); parse!(Thumbnail, k, v); + parse!(Resize, k, v); parse!(Blur, k, v); debug!("Skipping {}: {}, invalid", k, v); diff --git a/src/upload_manager.rs b/src/upload_manager.rs index 8eebf14..086dd8a 100644 --- a/src/upload_manager.rs +++ b/src/upload_manager.rs @@ -1,7 +1,7 @@ use crate::{ config::Format, error::UploadError, - migrate::{alias_key_bounds, variant_key_bounds, LatestDb, alias_id_key, alias_key}, + migrate::{alias_id_key, alias_key, alias_key_bounds, variant_key_bounds, LatestDb}, to_ext, validate::validate_image, }; @@ -46,6 +46,23 @@ impl std::fmt::Debug for UploadManager { type UploadStream = Pin>>>; +#[derive(serde::Deserialize, serde::Serialize)] +pub(crate) struct Details { + created_at: time::OffsetDateTime, +} + +impl Details { + pub(crate) fn now() -> Self { + Details { + created_at: time::OffsetDateTime::now_utc(), + } + } + + pub(crate) fn system_time(&self) -> std::time::SystemTime { + self.created_at.into() + } +} + struct FilenameIVec { inner: sled::IVec, } @@ -149,6 +166,55 @@ impl UploadManager { Ok(()) } + /// Get the image details for a given variant + pub(crate) async fn variant_details( + &self, + path: PathBuf, + filename: String, + ) -> Result, UploadError> { + let path_string = path.to_str().ok_or(UploadError::Path)?.to_string(); + + let fname_tree = self.inner.filename_tree.clone(); + debug!("Getting hash"); + let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes())) + .await? + .ok_or(UploadError::MissingFilename)?; + + let key = variant_details_key(&hash, &path_string); + let main_tree = self.inner.main_tree.clone(); + debug!("Getting details"); + let opt = match web::block(move || main_tree.get(key)).await? { + Some(ivec) => Some(serde_json::from_slice(&ivec)?), + None => None, + }; + debug!("Got details"); + + Ok(opt) + } + + pub(crate) async fn store_variant_details( + &self, + path: PathBuf, + filename: String, + ) -> Result<(), UploadError> { + let path_string = path.to_str().ok_or(UploadError::Path)?.to_string(); + + let fname_tree = self.inner.filename_tree.clone(); + debug!("Getting hash"); + let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes())) + .await? + .ok_or(UploadError::MissingFilename)?; + + let key = variant_details_key(&hash, &path_string); + let main_tree = self.inner.main_tree.clone(); + let details_value = serde_json::to_string(&Details::now())?; + debug!("Storing details"); + web::block(move || main_tree.insert(key, details_value.as_bytes())).await?; + debug!("Stored details"); + + Ok(()) + } + /// Get a list of aliases for a given file pub(crate) async fn aliases_by_filename( &self, @@ -470,9 +536,13 @@ impl UploadManager { for key in keys { let main_tree = self.inner.main_tree.clone(); if let Some(path) = web::block(move || main_tree.remove(key)).await? { - debug!("Deleting {:?}", String::from_utf8(path.to_vec())); - if let Err(e) = remove_path(path).await { - errors.push(e); + let s = String::from_utf8_lossy(&path); + debug!("Deleting {}", s); + // ignore json objects + if !s.starts_with('{') { + if let Err(e) = remove_path(path).await { + errors.push(e); + } } } } @@ -762,3 +832,11 @@ fn variant_key(hash: &[u8], path: &str) -> Vec { key.extend(path.as_bytes()); key } + +fn variant_details_key(hash: &[u8], path: &str) -> Vec { + let mut key = hash.to_vec(); + key.extend(&[2]); + key.extend(path.as_bytes()); + key.extend(b"details"); + key +}