diff --git a/Cargo.lock b/Cargo.lock index 1f5165c..e7a6f2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -361,15 +361,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" - -[[package]] -name = "array-init" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30bbe2f5e3d117f55bd8c7a1f9191e4a5deba9f15f595bbea4f670c59c765db" +checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4" [[package]] name = "async-trait" @@ -564,9 +558,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cexpr" @@ -628,20 +622,11 @@ dependencies = [ "vec_map", ] -[[package]] -name = "cloudabi" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" -dependencies = [ - "bitflags", -] - [[package]] name = "const_fn" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" [[package]] name = "cookie" @@ -675,21 +660,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", -] - [[package]] name = "crossbeam-epoch" version = "0.9.1" @@ -698,23 +668,12 @@ checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" dependencies = [ "cfg-if 1.0.0", "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils", "lazy_static", - "memoffset 0.6.1", + "memoffset", "scopeguard", ] -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", -] - [[package]] name = "crossbeam-utils" version = "0.8.1" @@ -1208,9 +1167,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" [[package]] name = "libloading" @@ -1299,27 +1258,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.1" @@ -1389,9 +1333,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.36" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" dependencies = [ "cfg-if 0.1.10", "libc", @@ -1479,12 +1423,11 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" dependencies = [ - "cfg-if 0.1.10", - "cloudabi", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -1506,7 +1449,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pict-rs" -version = "0.2.6" +version = "0.3.0-alpha.0" dependencies = [ "actix-form-data", "actix-fs", @@ -1526,10 +1469,10 @@ dependencies = [ "serde", "serde_json", "sha2", - "sled 0.32.1", - "sled 0.34.6", + "sled", "structopt", "thiserror", + "time 0.2.23", "tracing", "tracing-futures", "tracing-subscriber", @@ -1856,18 +1799,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" dependencies = [ "proc-macro2", "quote", @@ -1960,24 +1903,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" -[[package]] -name = "sled" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3dbbb8ee10611bd1d020767c27599ccbbf8365f7e0ed7e54429cc8b9433ad8" -dependencies = [ - "array-init", - "backtrace", - "crc32fast", - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot", -] - [[package]] name = "sled" version = "0.34.6" @@ -1985,8 +1910,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" dependencies = [ "crc32fast", - "crossbeam-epoch 0.9.1", - "crossbeam-utils 0.8.1", + "crossbeam-epoch", + "crossbeam-utils", "fs2", "fxhash", "libc", @@ -1996,9 +1921,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" [[package]] name = "socket2" @@ -2108,9 +2033,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.53" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68" +checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" dependencies = [ "proc-macro2", "quote", @@ -2192,6 +2117,7 @@ checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" dependencies = [ "const_fn", "libc", + "serde", "standback", "stdweb", "time-macros", @@ -2239,9 +2165,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" dependencies = [ "bytes", "fnv", diff --git a/Cargo.toml b/Cargo.toml index bb53237..b3a0fca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pict-rs" description = "A simple image hosting service" -version = "0.2.6" +version = "0.3.0-alpha.0" authors = ["asonix "] license = "AGPL-3.0" readme = "README.md" @@ -27,10 +27,10 @@ rexiv2 = "0.9.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.9.0" -sled032 = { version = "0.32.0", package = "sled" } sled = { version = "0.34.4" } structopt = "0.3.14" thiserror = "1.0" +time = { version = "0.2.23", features = ["serde"] } tracing = "0.1.15" tracing-futures = "0.2.4" tracing-subscriber = { version = "0.2.5", features = ["fmt", "tracing-log"] } diff --git a/README.md b/README.md index ab9f783..ade539e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ _a simple image hosting service_ ## Usage ### Running ``` -pict-rs 0.2.0-alpha.3 +pict-rs 0.3.0-alpha.0 USAGE: pict-rs [FLAGS] [OPTIONS] --path @@ -109,7 +109,10 @@ pict-rs offers the following endpoints: existing transformations include - `identity=true`: apply no changes - `blur={float}`: apply a gaussian blur to the file - - `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square + - `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` + square using raw pixel sampling + - `resize={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square + using a Lanczos2 filter. This is slower than sampling but looks a bit better in some cases Supported `ext` file extensions include `png`, `jpg`, and `webp` diff --git a/src/error.rs b/src/error.rs index c8fafb4..3f94780 100644 --- a/src/error.rs +++ b/src/error.rs @@ -9,9 +9,6 @@ pub(crate) enum UploadError { #[error("Couldn't save file, {0}")] Save(#[from] actix_fs::Error), - #[error("Error in DB migration, {0}")] - Migrate(#[from] sled032::Error), - #[error("Error in DB, {0}")] Db(#[from] sled::Error), @@ -71,6 +68,9 @@ pub(crate) enum UploadError { #[error("Error in MagickWand, {0}")] Wand(String), + + #[error("{0}")] + Json(#[from] serde_json::Error), } impl From for UploadError { diff --git a/src/main.rs b/src/main.rs index eff8553..5fbf2dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,7 +26,7 @@ use self::{ error::UploadError, middleware::{Internal, Tracing}, processor::process_image, - upload_manager::UploadManager, + upload_manager::{Details, UploadManager}, validate::{image_webp, video_mp4}, }; @@ -289,38 +289,28 @@ async fn process( .map_err(|_| UploadError::UnsupportedFormat)?; let processed_name = format!("{}.{}", name, ext); let base = manager.image_dir(); - let mut path = self::processor::build_path(base, &chain, processed_name); - - if let Some((updated_path, exists)) = self::processor::prepare_image(path.clone()).await? { - path = updated_path.clone(); - - if exists.is_new() { - // Save the transcoded file in another task - debug!("Spawning storage task"); - let span = Span::current(); - let manager2 = manager.clone(); - let name = name.clone(); - actix_rt::spawn(async move { - let entered = span.enter(); - if let Err(e) = manager2.store_variant(updated_path, name).await { - error!("Error storing variant, {}", e); - return; - } - drop(entered); - }); - } - } + let thumbnail_path = self::processor::build_path(base, &chain, processed_name); // If the thumbnail doesn't exist, we need to create it - if let Err(e) = actix_fs::metadata(path.clone()).await { + let thumbnail_exists = if let Err(e) = actix_fs::metadata(thumbnail_path.clone()).await { if e.kind() != Some(std::io::ErrorKind::NotFound) { error!("Error looking up processed image, {}", e); return Err(e.into()); } + false + } else { + true + }; + let details = manager + .variant_details(thumbnail_path.clone(), name.clone()) + .await?; + + if !thumbnail_exists || details.is_none() { let mut original_path = manager.image_dir(); original_path.push(name.clone()); + // Create and save a JPG for motion images (gif, mp4) if let Some((updated_path, exists)) = self::processor::prepare_image(original_path.clone()).await? { @@ -346,14 +336,25 @@ async fn process( // apply chain to the provided image let img_bytes = process_image(original_path.clone(), chain, format).await?; - let path2 = path.clone(); + let path2 = thumbnail_path.clone(); let img_bytes2 = img_bytes.clone(); // Save the file in another task, we want to return the thumbnail now debug!("Spawning storage task"); let span = Span::current(); + let store_details = details.is_none(); actix_rt::spawn(async move { let entered = span.enter(); + if store_details { + debug!("Storing details"); + if let Err(e) = manager + .store_variant_details(path2.clone(), name.clone()) + .await + { + error!("Error storing details, {}", e); + return; + } + } if let Err(e) = manager.store_variant(path2.clone(), name).await { error!("Error storing variant, {}", e); return; @@ -365,23 +366,27 @@ async fn process( drop(entered); }); + let details = details.unwrap_or(Details::now()); + return Ok(srv_response( Box::pin(futures::stream::once(async { Ok(img_bytes) as Result<_, UploadError> })), content_type, 7 * DAYS, - SystemTime::now(), + details.system_time(), )); } - let stream = actix_fs::read_to_stream(path).await?; + let stream = actix_fs::read_to_stream(thumbnail_path).await?; + + let details = details.unwrap_or(Details::now()); Ok(srv_response( stream, content_type, 7 * DAYS, - SystemTime::now(), + details.system_time(), )) } @@ -394,7 +399,15 @@ async fn serve( let name = manager.from_alias(alias.into_inner()).await?; let content_type = from_name(&name)?; let mut path = manager.image_dir(); - path.push(name); + path.push(name.clone()); + + let details = manager.variant_details(path.clone(), name.clone()).await?; + + if details.is_none() { + manager.store_variant_details(path.clone(), name).await?; + } + + let details = details.unwrap_or(Details::now()); let stream = actix_fs::read_to_stream(path).await?; @@ -402,7 +415,7 @@ async fn serve( stream, content_type, 7 * DAYS, - SystemTime::now(), + details.system_time(), )) } diff --git a/src/migrate/mod.rs b/src/migrate/mod.rs index d82fb7a..99d1108 100644 --- a/src/migrate/mod.rs +++ b/src/migrate/mod.rs @@ -1,9 +1,7 @@ use crate::UploadError; use sled; use std::path::PathBuf; -use tracing::{debug, info, warn}; -mod s032; mod s034; type SledIter = Box, Vec), UploadError>>>; @@ -68,9 +66,7 @@ impl LatestDb { loop { let root_dir2 = root_dir.clone(); - let res = std::panic::catch_unwind(move || { - version.migrate(root_dir2) - }); + let res = std::panic::catch_unwind(move || version.migrate(root_dir2)); if let Ok(res) = res { return res; @@ -81,8 +77,6 @@ impl LatestDb { #[derive(Clone, Copy)] enum DbVersion { - Sled0320Rc1, - Sled032, Sled034, Fresh, } @@ -93,170 +87,16 @@ impl DbVersion { return DbVersion::Sled034; } - if s032::exists(root.clone()) { - return DbVersion::Sled032; - } - - if s032::exists_rc1(root.clone()) { - return DbVersion::Sled0320Rc1; - } - DbVersion::Fresh } fn migrate(self, root: PathBuf) -> Result { match self { - DbVersion::Sled0320Rc1 => migrate_0_32_0_rc1(root), - DbVersion::Sled032 => migrate_0_32(root), DbVersion::Sled034 | DbVersion::Fresh => s034::open(root), } } } -fn migrate_0_32_0_rc1(root: PathBuf) -> Result { - info!("Migrating database from 0.32.0-rc1 to 0.34.0"); - - let old_db = s032::open_rc1(root.clone())?; - let new_db = s034::open(root)?; - - migrate(old_db, new_db) -} - -fn migrate_0_32(root: PathBuf) -> Result { - info!("Migrating database from 0.32 to 0.34"); - - let old_db = s032::open(root.clone())?; - let new_db = s034::open(root)?; - - migrate(old_db, new_db) -} - -fn migrate(old_db: Old, new_db: New) -> Result -where - Old: SledDb, - New: SledDb, -{ - let old_alias_tree = old_db.open_tree("alias")?; - let new_alias_tree = new_db.open_tree("alias")?; - - let new_migrate_tree = new_db.open_tree("migrate")?; - if let Some(_) = new_migrate_tree.get("done")? { - return Ok(new_db); - } - - let (iterator, mut counter) = if let Some(last_migrated) = new_migrate_tree.get("last_migrated")? { - let mut last_migrated = String::from_utf8_lossy(&last_migrated).to_string(); - info!("Previous migration failed after {}, attempting to skip", last_migrated); - if let Some(index) = last_migrated.find('.') { - last_migrated = last_migrated.split_at(index).0.to_owned(); - } - if last_migrated.len() > 3 { - last_migrated = last_migrated.split_at(3).0.to_owned(); - } - let last_migrated = increment_alphanumeric(&last_migrated).as_bytes().to_owned(); - new_migrate_tree.insert("last_migrated", last_migrated.clone())?; - new_migrate_tree.flush()?; - if let Some(count) = new_migrate_tree.get("counter")? { - (old_alias_tree.range(last_migrated..), String::from_utf8_lossy(&count).parse::().unwrap()) - } else { - (old_alias_tree.range(last_migrated..), 0) - } - } else { - (old_alias_tree.iter(), 0) - }; - - for res in iterator { - let (k, _) = res?; - - if let Some(v) = old_alias_tree.get(&k)? { - if !k.contains(&b"/"[0]) { - if let Some(id) = old_alias_tree.get(alias_id_key(&String::from_utf8_lossy(&k)))? { - counter += 1; - debug!("Migrating alias #{}", counter); - // k is an alias - migrate_main_tree(&k, &v, &old_db, &new_db, &String::from_utf8_lossy(&id))?; - debug!( - "Moving alias -> hash for alias {}", - String::from_utf8_lossy(k.as_ref()), - ); - new_migrate_tree.insert("counter", format!("{}", counter))?; - } - } else { - debug!( - "Moving {}, {}", - String::from_utf8_lossy(k.as_ref()), - String::from_utf8_lossy(v.as_ref()) - ); - } - new_alias_tree.insert(k.clone(), v)?; - new_migrate_tree.insert("last_migrated", k)?; - new_alias_tree.flush()?; - new_migrate_tree.flush()?; - } else { - warn!("MISSING {}", String::from_utf8_lossy(k.as_ref())); - } - } - info!("Moved {} unique aliases", counter); - - new_migrate_tree.insert("done", "true")?; - new_migrate_tree.flush()?; - - Ok(new_db) -} - -fn migrate_main_tree( - alias: &[u8], - hash: &[u8], - old_db: Old, - new_db: New, - id: &str, -) -> Result<(), UploadError> -where - Old: SledDb, - New: SledDb, -{ - let main_tree = new_db.open_tree("main")?; - - let new_fname_tree = new_db.open_tree("filename")?; - - debug!( - "Migrating files for {}", - String::from_utf8_lossy(alias.as_ref()) - ); - if let Some(filename) = old_db.self_tree().get(&hash)? { - main_tree.insert(&hash, filename.clone())?; - new_fname_tree.insert(filename, hash.clone())?; - main_tree.flush()?; - new_fname_tree.flush()?; - } else { - warn!("Missing filename"); - } - - let key = alias_key(&hash, id); - if let Some(v) = old_db.self_tree().get(&key)? { - main_tree.insert(key, v)?; - main_tree.flush()?; - } else { - warn!("Not migrating alias {} id {}", String::from_utf8_lossy(&alias), id); - return Ok(()); - } - - let (start, end) = variant_key_bounds(&hash); - if main_tree.range(start.clone()..end.clone()).next().is_none() { - let mut counter = 0; - for res in old_db.self_tree().range(start.clone()..end.clone()) { - counter += 1; - let (k, v) = res?; - debug!("Moving variant #{} for {}", counter, String::from_utf8_lossy(v.as_ref())); - main_tree.insert(k, v)?; - main_tree.flush()?; - } - debug!("Moved {} variants for {}", counter, String::from_utf8_lossy(alias.as_ref())); - } - - Ok(()) -} - pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec, Vec) { let mut start = hash.to_vec(); start.extend(&[0]); @@ -289,52 +129,3 @@ pub(crate) fn alias_key(hash: &[u8], id: &str) -> Vec { key } - -const VALID: &[char] = &[ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', -]; - -fn increment_alphanumeric(input: &str) -> String { - let (_, output) = input.chars().rev().fold((true, String::new()), |(incr_next, mut acc), item| { - if incr_next { - let mut index = None; - for (i, test) in VALID.iter().enumerate() { - if *test == item { - index = Some(i); - } - } - let index = index.unwrap_or(0); - let (set_incr_next, next_index) = if index == (VALID.len() - 1) { - (true, 0) - } else { - (false, index + 1) - }; - acc.extend(&[VALID[next_index]]); - (set_incr_next, acc) - } else { - acc.extend(&[item]); - (false, acc) - } - }); - - output.chars().rev().collect() -} - -#[cfg(test)] -mod tests { - use super::increment_alphanumeric; - - #[test] - fn increments() { - assert_eq!(increment_alphanumeric("hello"), "hellp"); - assert_eq!(increment_alphanumeric("0"), "1"); - assert_eq!(increment_alphanumeric("9"), "A"); - assert_eq!(increment_alphanumeric("Z"), "a"); - assert_eq!(increment_alphanumeric("z"), "0"); - assert_eq!(increment_alphanumeric("az"), "b0"); - assert_eq!(increment_alphanumeric("19"), "1A"); - assert_eq!(increment_alphanumeric("AZ"), "Aa"); - } -} diff --git a/src/migrate/s032.rs b/src/migrate/s032.rs deleted file mode 100644 index 732b9dc..0000000 --- a/src/migrate/s032.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::{ - migrate::{SledDb, SledIter, SledTree}, - UploadError, -}; -use std::path::PathBuf; - -const SLED_032: &str = "db-0.32"; -const SLED_0320_RC1: &str = "db"; - -pub(crate) fn exists_rc1(mut base: PathBuf) -> bool { - base.push(SLED_0320_RC1); - - std::fs::metadata(base).is_ok() -} - -pub(crate) fn open_rc1(mut base: PathBuf) -> Result { - base.push(SLED_0320_RC1); - - Ok(sled032::open(base)?) -} - -pub(crate) fn exists(mut base: PathBuf) -> bool { - base.push("sled"); - base.push(SLED_032); - - std::fs::metadata(base).is_ok() -} - -pub(crate) fn open(mut base: PathBuf) -> Result { - base.push("sled"); - base.push(SLED_032); - - Ok(sled032::open(base)?) -} - -impl SledDb for sled032::Db { - type SledTree = sled032::Tree; - - fn open_tree(&self, name: &str) -> Result { - Ok(sled032::Db::open_tree(self, name)?) - } - - fn self_tree(&self) -> &Self::SledTree { - &*self - } -} - -impl SledTree for sled032::Tree { - fn get(&self, key: K) -> Result>, UploadError> - where - K: AsRef<[u8]>, - { - Ok(sled032::Tree::get(self, key)?.map(|v| Vec::from(v.as_ref()))) - } - - fn insert(&self, key: K, value: V) -> Result<(), UploadError> - where - K: AsRef<[u8]>, - V: AsRef<[u8]>, - { - Ok(sled032::Tree::insert(self, key, value.as_ref().to_vec()).map(|_| ())?) - } - - fn iter(&self) -> SledIter { - Box::new(sled032::Tree::iter(self).map(|res| { - res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec())) - .map_err(UploadError::from) - })) - } - - fn range(&self, range: R) -> SledIter - where - K: AsRef<[u8]>, - R: std::ops::RangeBounds, - { - Box::new(sled032::Tree::range(self, range).map(|res| { - res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec())) - .map_err(UploadError::from) - })) - } - - fn flush(&self) -> Result<(), UploadError> { - sled032::Tree::flush(self).map(|_| ()).map_err(UploadError::from) - } -} diff --git a/src/processor.rs b/src/processor.rs index 3c49021..72fee67 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -113,6 +113,63 @@ impl Processor for Thumbnail { } } +pub(crate) struct Resize(usize); + +impl Processor for Resize { + fn name() -> &'static str + where + Self: Sized, + { + "resize" + } + + fn is_processor(s: &str) -> bool + where + Self: Sized, + { + s == Self::name() + } + + fn parse(_: &str, v: &str) -> Option> + where + Self: Sized, + { + let size = v.parse().ok()?; + Some(Box::new(Resize(size))) + } + + fn path(&self, mut path: PathBuf) -> PathBuf { + path.push(Self::name()); + path.push(self.0.to_string()); + path + } + + fn process(&self, wand: &mut MagickWand) -> Result<(), UploadError> { + debug!("Resize"); + let width = wand.get_image_width(); + let height = wand.get_image_height(); + + if width > self.0 || height > self.0 { + let width_ratio = width as f64 / self.0 as f64; + let height_ratio = height as f64 / self.0 as f64; + + let (new_width, new_height) = if width_ratio < height_ratio { + (width as f64 / height_ratio, self.0 as f64) + } else { + (self.0 as f64, height as f64 / width_ratio) + }; + + wand.resize_image( + new_width as usize, + new_height as usize, + magick_rust::bindings::FilterType_Lanczos2Filter, + ); + } + + Ok(()) + } +} + pub(crate) struct Blur(f64); impl Processor for Blur { @@ -178,6 +235,7 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> ProcessChain { parse!(Identity, k, v); parse!(Thumbnail, k, v); + parse!(Resize, k, v); parse!(Blur, k, v); debug!("Skipping {}: {}, invalid", k, v); diff --git a/src/upload_manager.rs b/src/upload_manager.rs index 8eebf14..086dd8a 100644 --- a/src/upload_manager.rs +++ b/src/upload_manager.rs @@ -1,7 +1,7 @@ use crate::{ config::Format, error::UploadError, - migrate::{alias_key_bounds, variant_key_bounds, LatestDb, alias_id_key, alias_key}, + migrate::{alias_id_key, alias_key, alias_key_bounds, variant_key_bounds, LatestDb}, to_ext, validate::validate_image, }; @@ -46,6 +46,23 @@ impl std::fmt::Debug for UploadManager { type UploadStream = Pin>>>; +#[derive(serde::Deserialize, serde::Serialize)] +pub(crate) struct Details { + created_at: time::OffsetDateTime, +} + +impl Details { + pub(crate) fn now() -> Self { + Details { + created_at: time::OffsetDateTime::now_utc(), + } + } + + pub(crate) fn system_time(&self) -> std::time::SystemTime { + self.created_at.into() + } +} + struct FilenameIVec { inner: sled::IVec, } @@ -149,6 +166,55 @@ impl UploadManager { Ok(()) } + /// Get the image details for a given variant + pub(crate) async fn variant_details( + &self, + path: PathBuf, + filename: String, + ) -> Result, UploadError> { + let path_string = path.to_str().ok_or(UploadError::Path)?.to_string(); + + let fname_tree = self.inner.filename_tree.clone(); + debug!("Getting hash"); + let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes())) + .await? + .ok_or(UploadError::MissingFilename)?; + + let key = variant_details_key(&hash, &path_string); + let main_tree = self.inner.main_tree.clone(); + debug!("Getting details"); + let opt = match web::block(move || main_tree.get(key)).await? { + Some(ivec) => Some(serde_json::from_slice(&ivec)?), + None => None, + }; + debug!("Got details"); + + Ok(opt) + } + + pub(crate) async fn store_variant_details( + &self, + path: PathBuf, + filename: String, + ) -> Result<(), UploadError> { + let path_string = path.to_str().ok_or(UploadError::Path)?.to_string(); + + let fname_tree = self.inner.filename_tree.clone(); + debug!("Getting hash"); + let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes())) + .await? + .ok_or(UploadError::MissingFilename)?; + + let key = variant_details_key(&hash, &path_string); + let main_tree = self.inner.main_tree.clone(); + let details_value = serde_json::to_string(&Details::now())?; + debug!("Storing details"); + web::block(move || main_tree.insert(key, details_value.as_bytes())).await?; + debug!("Stored details"); + + Ok(()) + } + /// Get a list of aliases for a given file pub(crate) async fn aliases_by_filename( &self, @@ -470,9 +536,13 @@ impl UploadManager { for key in keys { let main_tree = self.inner.main_tree.clone(); if let Some(path) = web::block(move || main_tree.remove(key)).await? { - debug!("Deleting {:?}", String::from_utf8(path.to_vec())); - if let Err(e) = remove_path(path).await { - errors.push(e); + let s = String::from_utf8_lossy(&path); + debug!("Deleting {}", s); + // ignore json objects + if !s.starts_with('{') { + if let Err(e) = remove_path(path).await { + errors.push(e); + } } } } @@ -762,3 +832,11 @@ fn variant_key(hash: &[u8], path: &str) -> Vec { key.extend(path.as_bytes()); key } + +fn variant_details_key(hash: &[u8], path: &str) -> Vec { + let mut key = hash.to_vec(); + key.extend(&[2]); + key.extend(path.as_bytes()); + key.extend(b"details"); + key +}