LemmyNet
/
pict-rs
mirror of https://git.asonix.dog/asonix/pict-rs
2
0
Fork 0
Code Releases Activity

Add initial details implementation 

Details is an object generated for each image, containing
it's created_at date to properly set response headers. It
also can be expanding in the future to include an image's
dimensions, which we can expose via an API endpoint.

In addition to this, Details allows us the ability to
re-generate processed images. For now, all images which
do not yet have an associated Details will be
re-generated upon access. Details is set for each
thumbnail on generation, so this will only happen once
for each image.
This commit is contained in:
asonix 2020-12-09 23:05:04 -06:00
parent e3dbd5e791
commit 0bf7e0f688
9 changed files with 224 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
Cargo.lock generated
View File

@ -361,15 +361,9 @@ dependencies = [
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.34" version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4"
[[package]]
name = "array-init"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f30bbe2f5e3d117f55bd8c7a1f9191e4a5deba9f15f595bbea4f670c59c765db"
[[package]] [[package]]
name = "async-trait" name = "async-trait"
@ -564,9 +558,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.65" version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
[[package]] [[package]]
name = "cexpr" name = "cexpr"
@ -628,20 +622,11 @@ dependencies = [
"vec_map", "vec_map",
] ]
[[package]]
name = "cloudabi"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
dependencies = [
"bitflags",
]
[[package]] [[package]]
name = "const_fn" name = "const_fn"
version = "0.4.3" version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826"
[[package]] [[package]]
name = "cookie" name = "cookie"
@ -675,21 +660,6 @@ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
] ]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg",
"cfg-if 0.1.10",
"crossbeam-utils 0.7.2",
"lazy_static",
"maybe-uninit",
"memoffset 0.5.6",
"scopeguard",
]
[[package]] [[package]]
name = "crossbeam-epoch" name = "crossbeam-epoch"
version = "0.9.1" version = "0.9.1"
@ -698,23 +668,12 @@ checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"const_fn", "const_fn",
"crossbeam-utils 0.8.1", "crossbeam-utils",
"lazy_static", "lazy_static",
"memoffset 0.6.1", "memoffset",
"scopeguard", "scopeguard",
] ]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"cfg-if 0.1.10",
"lazy_static",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.1" version = "0.8.1"
@ -1208,9 +1167,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.80" version = "0.2.81"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
[[package]] [[package]]
name = "libloading" name = "libloading"
@ -1299,27 +1258,12 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.3.4" version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memoffset"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "memoffset" name = "memoffset"
version = "0.6.1" version = "0.6.1"
@ -1389,9 +1333,9 @@ dependencies = [
[[package]] [[package]]
name = "net2" name = "net2"
version = "0.2.36" version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02" checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
dependencies = [ dependencies = [
"cfg-if 0.1.10", "cfg-if 0.1.10",
"libc", "libc",
@ -1479,12 +1423,11 @@ dependencies = [
[[package]] [[package]]
name = "parking_lot_core" name = "parking_lot_core"
version = "0.8.0" version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0"
dependencies = [ dependencies = [
"cfg-if 0.1.10", "cfg-if 1.0.0",
"cloudabi",
"instant", "instant",
"libc", "libc",
"redox_syscall", "redox_syscall",
@ -1506,7 +1449,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]] [[package]]
name = "pict-rs" name = "pict-rs"
version = "0.2.6" version = "0.3.0-alpha.0"
dependencies = [ dependencies = [
"actix-form-data", "actix-form-data",
"actix-fs", "actix-fs",
@ -1526,10 +1469,10 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"sha2", "sha2",
"sled 0.32.1", "sled",
"sled 0.34.6",
"structopt", "structopt",
"thiserror", "thiserror",
"time 0.2.23",
"tracing", "tracing",
"tracing-futures", "tracing-futures",
"tracing-subscriber", "tracing-subscriber",
@ -1856,18 +1799,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.117" version = "1.0.118"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800"
dependencies = [ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.117" version = "1.0.118"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -1960,24 +1903,6 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
[[package]]
name = "sled"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3dbbb8ee10611bd1d020767c27599ccbbf8365f7e0ed7e54429cc8b9433ad8"
dependencies = [
"array-init",
"backtrace",
"crc32fast",
"crossbeam-epoch 0.8.2",
"crossbeam-utils 0.7.2",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot",
]
[[package]] [[package]]
name = "sled" name = "sled"
version = "0.34.6" version = "0.34.6"
@ -1985,8 +1910,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
dependencies = [ dependencies = [
"crc32fast", "crc32fast",
"crossbeam-epoch 0.9.1", "crossbeam-epoch",
"crossbeam-utils 0.8.1", "crossbeam-utils",
"fs2", "fs2",
"fxhash", "fxhash",
"libc", "libc",
@ -1996,9 +1921,9 @@ dependencies = [
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.5.0" version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85" checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75"
[[package]] [[package]]
name = "socket2" name = "socket2"
@ -2108,9 +2033,9 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.53" version = "1.0.54"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68" checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2192,6 +2117,7 @@ checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b"
dependencies = [ dependencies = [
"const_fn", "const_fn",
"libc", "libc",
"serde",
"standback", "standback",
"stdweb", "stdweb",
"time-macros", "time-macros",
@ -2239,9 +2165,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "0.2.23" version = "0.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff" checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48"
dependencies = [ dependencies = [
"bytes", "bytes",
"fnv", "fnv",

4
Cargo.toml
View File

@ -1,7 +1,7 @@
[package] [package]
name = "pict-rs" name = "pict-rs"
description = "A simple image hosting service" description = "A simple image hosting service"
version = "0.2.6" version = "0.3.0-alpha.0"
authors = ["asonix <asonix@asonix.dog>"] authors = ["asonix <asonix@asonix.dog>"]
license = "AGPL-3.0" license = "AGPL-3.0"
readme = "README.md" readme = "README.md"
@ -27,10 +27,10 @@ rexiv2 = "0.9.1"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
sha2 = "0.9.0" sha2 = "0.9.0"
sled032 = { version = "0.32.0", package = "sled" }
sled = { version = "0.34.4" } sled = { version = "0.34.4" }
structopt = "0.3.14" structopt = "0.3.14"
thiserror = "1.0" thiserror = "1.0"
time = { version = "0.2.23", features = ["serde"] }
tracing = "0.1.15" tracing = "0.1.15"
tracing-futures = "0.2.4" tracing-futures = "0.2.4"
tracing-subscriber = { version = "0.2.5", features = ["fmt", "tracing-log"] } tracing-subscriber = { version = "0.2.5", features = ["fmt", "tracing-log"] }

7
README.md
View File

@ -4,7 +4,7 @@ _a simple image hosting service_
## Usage ## Usage
### Running ### Running
``` ```
pict-rs 0.2.0-alpha.3 pict-rs 0.3.0-alpha.0
USAGE: USAGE:
pict-rs [FLAGS] [OPTIONS] --path <path> pict-rs [FLAGS] [OPTIONS] --path <path>
@ -109,7 +109,10 @@ pict-rs offers the following endpoints:
existing transformations include existing transformations include
- `identity=true`: apply no changes - `identity=true`: apply no changes
- `blur={float}`: apply a gaussian blur to the file - `blur={float}`: apply a gaussian blur to the file
- `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square - `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}`
square using raw pixel sampling
- `resize={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square
using a Lanczos2 filter. This is slower than sampling but looks a bit better in some cases
Supported `ext` file extensions include `png`, `jpg`, and `webp` Supported `ext` file extensions include `png`, `jpg`, and `webp`

6
src/error.rs
View File

@ -9,9 +9,6 @@ pub(crate) enum UploadError {
#[error("Couldn't save file, {0}")] #[error("Couldn't save file, {0}")]
Save(#[from] actix_fs::Error), Save(#[from] actix_fs::Error),
#[error("Error in DB migration, {0}")]
Migrate(#[from] sled032::Error),
#[error("Error in DB, {0}")] #[error("Error in DB, {0}")]
Db(#[from] sled::Error), Db(#[from] sled::Error),
@ -71,6 +68,9 @@ pub(crate) enum UploadError {
#[error("Error in MagickWand, {0}")] #[error("Error in MagickWand, {0}")]
Wand(String), Wand(String),
#[error("{0}")]
Json(#[from] serde_json::Error),
} }
impl From<actix_web::client::SendRequestError> for UploadError { impl From<actix_web::client::SendRequestError> for UploadError {

71
src/main.rs
View File

@ -26,7 +26,7 @@ use self::{
error::UploadError, error::UploadError,
middleware::{Internal, Tracing}, middleware::{Internal, Tracing},
processor::process_image, processor::process_image,
upload_manager::UploadManager, upload_manager::{Details, UploadManager},
validate::{image_webp, video_mp4}, validate::{image_webp, video_mp4},
}; };
@ -289,38 +289,28 @@ async fn process(
.map_err(|_| UploadError::UnsupportedFormat)?; .map_err(|_| UploadError::UnsupportedFormat)?;
let processed_name = format!("{}.{}", name, ext); let processed_name = format!("{}.{}", name, ext);
let base = manager.image_dir(); let base = manager.image_dir();
let mut path = self::processor::build_path(base, &chain, processed_name); let thumbnail_path = self::processor::build_path(base, &chain, processed_name);
if let Some((updated_path, exists)) = self::processor::prepare_image(path.clone()).await? {
path = updated_path.clone();
if exists.is_new() {
// Save the transcoded file in another task
debug!("Spawning storage task");
let span = Span::current();
let manager2 = manager.clone();
let name = name.clone();
actix_rt::spawn(async move {
let entered = span.enter();
if let Err(e) = manager2.store_variant(updated_path, name).await {
error!("Error storing variant, {}", e);
return;
}
drop(entered);
});
}
}
// If the thumbnail doesn't exist, we need to create it // If the thumbnail doesn't exist, we need to create it
if let Err(e) = actix_fs::metadata(path.clone()).await { let thumbnail_exists = if let Err(e) = actix_fs::metadata(thumbnail_path.clone()).await {
if e.kind() != Some(std::io::ErrorKind::NotFound) { if e.kind() != Some(std::io::ErrorKind::NotFound) {
error!("Error looking up processed image, {}", e); error!("Error looking up processed image, {}", e);
return Err(e.into()); return Err(e.into());
} }
false
} else {
true
};
let details = manager
.variant_details(thumbnail_path.clone(), name.clone())
.await?;
if !thumbnail_exists || details.is_none() {
let mut original_path = manager.image_dir(); let mut original_path = manager.image_dir();
original_path.push(name.clone()); original_path.push(name.clone());
// Create and save a JPG for motion images (gif, mp4)
if let Some((updated_path, exists)) = if let Some((updated_path, exists)) =
self::processor::prepare_image(original_path.clone()).await? self::processor::prepare_image(original_path.clone()).await?
{ {
@ -346,14 +336,25 @@ async fn process(
// apply chain to the provided image // apply chain to the provided image
let img_bytes = process_image(original_path.clone(), chain, format).await?; let img_bytes = process_image(original_path.clone(), chain, format).await?;
let path2 = path.clone(); let path2 = thumbnail_path.clone();
let img_bytes2 = img_bytes.clone(); let img_bytes2 = img_bytes.clone();
// Save the file in another task, we want to return the thumbnail now // Save the file in another task, we want to return the thumbnail now
debug!("Spawning storage task"); debug!("Spawning storage task");
let span = Span::current(); let span = Span::current();
let store_details = details.is_none();
actix_rt::spawn(async move { actix_rt::spawn(async move {
let entered = span.enter(); let entered = span.enter();
if store_details {
debug!("Storing details");
if let Err(e) = manager
.store_variant_details(path2.clone(), name.clone())
.await
{
error!("Error storing details, {}", e);
return;
}
}
if let Err(e) = manager.store_variant(path2.clone(), name).await { if let Err(e) = manager.store_variant(path2.clone(), name).await {
error!("Error storing variant, {}", e); error!("Error storing variant, {}", e);
return; return;
@ -365,23 +366,27 @@ async fn process(
drop(entered); drop(entered);
}); });
let details = details.unwrap_or(Details::now());
return Ok(srv_response( return Ok(srv_response(
Box::pin(futures::stream::once(async { Box::pin(futures::stream::once(async {
Ok(img_bytes) as Result<_, UploadError> Ok(img_bytes) as Result<_, UploadError>
})), })),
content_type, content_type,
7 * DAYS, 7 * DAYS,
SystemTime::now(), details.system_time(),
)); ));
} }
let stream = actix_fs::read_to_stream(path).await?; let stream = actix_fs::read_to_stream(thumbnail_path).await?;
let details = details.unwrap_or(Details::now());
Ok(srv_response( Ok(srv_response(
stream, stream,
content_type, content_type,
7 * DAYS, 7 * DAYS,
SystemTime::now(), details.system_time(),
)) ))
} }
@ -394,7 +399,15 @@ async fn serve(
let name = manager.from_alias(alias.into_inner()).await?; let name = manager.from_alias(alias.into_inner()).await?;
let content_type = from_name(&name)?; let content_type = from_name(&name)?;
let mut path = manager.image_dir(); let mut path = manager.image_dir();
path.push(name); path.push(name.clone());
let details = manager.variant_details(path.clone(), name.clone()).await?;
if details.is_none() {
manager.store_variant_details(path.clone(), name).await?;
}
let details = details.unwrap_or(Details::now());
let stream = actix_fs::read_to_stream(path).await?; let stream = actix_fs::read_to_stream(path).await?;
@ -402,7 +415,7 @@ async fn serve(
stream, stream,
content_type, content_type,
7 * DAYS, 7 * DAYS,
SystemTime::now(), details.system_time(),
)) ))
} }

211
src/migrate/mod.rs
View File

@ -1,9 +1,7 @@
use crate::UploadError; use crate::UploadError;
use sled; use sled;
use std::path::PathBuf; use std::path::PathBuf;
use tracing::{debug, info, warn};
mod s032;
mod s034; mod s034;
type SledIter = Box<dyn Iterator<Item = Result<(Vec<u8>, Vec<u8>), UploadError>>>; type SledIter = Box<dyn Iterator<Item = Result<(Vec<u8>, Vec<u8>), UploadError>>>;
@ -68,9 +66,7 @@ impl LatestDb {
loop { loop {
let root_dir2 = root_dir.clone(); let root_dir2 = root_dir.clone();
let res = std::panic::catch_unwind(move || { let res = std::panic::catch_unwind(move || version.migrate(root_dir2));
version.migrate(root_dir2)
});
if let Ok(res) = res { if let Ok(res) = res {
return res; return res;
@ -81,8 +77,6 @@ impl LatestDb {
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
enum DbVersion { enum DbVersion {
Sled0320Rc1,
Sled032,
Sled034, Sled034,
Fresh, Fresh,
} }
@ -93,170 +87,16 @@ impl DbVersion {
return DbVersion::Sled034; return DbVersion::Sled034;
} }
if s032::exists(root.clone()) {
return DbVersion::Sled032;
}
if s032::exists_rc1(root.clone()) {
return DbVersion::Sled0320Rc1;
}
DbVersion::Fresh DbVersion::Fresh
} }
fn migrate(self, root: PathBuf) -> Result<sled::Db, UploadError> { fn migrate(self, root: PathBuf) -> Result<sled::Db, UploadError> {
match self { match self {
DbVersion::Sled0320Rc1 => migrate_0_32_0_rc1(root),
DbVersion::Sled032 => migrate_0_32(root),
DbVersion::Sled034 | DbVersion::Fresh => s034::open(root), DbVersion::Sled034 | DbVersion::Fresh => s034::open(root),
} }
} }
} }
fn migrate_0_32_0_rc1(root: PathBuf) -> Result<sled::Db, UploadError> {
info!("Migrating database from 0.32.0-rc1 to 0.34.0");
let old_db = s032::open_rc1(root.clone())?;
let new_db = s034::open(root)?;
migrate(old_db, new_db)
}
fn migrate_0_32(root: PathBuf) -> Result<sled::Db, UploadError> {
info!("Migrating database from 0.32 to 0.34");
let old_db = s032::open(root.clone())?;
let new_db = s034::open(root)?;
migrate(old_db, new_db)
}
fn migrate<Old, New>(old_db: Old, new_db: New) -> Result<New, UploadError>
where
Old: SledDb,
New: SledDb,
{
let old_alias_tree = old_db.open_tree("alias")?;
let new_alias_tree = new_db.open_tree("alias")?;
let new_migrate_tree = new_db.open_tree("migrate")?;
if let Some(_) = new_migrate_tree.get("done")? {
return Ok(new_db);
}
let (iterator, mut counter) = if let Some(last_migrated) = new_migrate_tree.get("last_migrated")? {
let mut last_migrated = String::from_utf8_lossy(&last_migrated).to_string();
info!("Previous migration failed after {}, attempting to skip", last_migrated);
if let Some(index) = last_migrated.find('.') {
last_migrated = last_migrated.split_at(index).0.to_owned();
}
if last_migrated.len() > 3 {
last_migrated = last_migrated.split_at(3).0.to_owned();
}
let last_migrated = increment_alphanumeric(&last_migrated).as_bytes().to_owned();
new_migrate_tree.insert("last_migrated", last_migrated.clone())?;
new_migrate_tree.flush()?;
if let Some(count) = new_migrate_tree.get("counter")? {
(old_alias_tree.range(last_migrated..), String::from_utf8_lossy(&count).parse::<usize>().unwrap())
} else {
(old_alias_tree.range(last_migrated..), 0)
}
} else {
(old_alias_tree.iter(), 0)
};
for res in iterator {
let (k, _) = res?;
if let Some(v) = old_alias_tree.get(&k)? {
if !k.contains(&b"/"[0]) {
if let Some(id) = old_alias_tree.get(alias_id_key(&String::from_utf8_lossy(&k)))? {
counter += 1;
debug!("Migrating alias #{}", counter);
// k is an alias
migrate_main_tree(&k, &v, &old_db, &new_db, &String::from_utf8_lossy(&id))?;
debug!(
"Moving alias -> hash for alias {}",
String::from_utf8_lossy(k.as_ref()),
);
new_migrate_tree.insert("counter", format!("{}", counter))?;
}
} else {
debug!(
"Moving {}, {}",
String::from_utf8_lossy(k.as_ref()),
String::from_utf8_lossy(v.as_ref())
);
}
new_alias_tree.insert(k.clone(), v)?;
new_migrate_tree.insert("last_migrated", k)?;
new_alias_tree.flush()?;
new_migrate_tree.flush()?;
} else {
warn!("MISSING {}", String::from_utf8_lossy(k.as_ref()));
}
}
info!("Moved {} unique aliases", counter);
new_migrate_tree.insert("done", "true")?;
new_migrate_tree.flush()?;
Ok(new_db)
}
fn migrate_main_tree<Old, New>(
alias: &[u8],
hash: &[u8],
old_db: Old,
new_db: New,
id: &str,
) -> Result<(), UploadError>
where
Old: SledDb,
New: SledDb,
{
let main_tree = new_db.open_tree("main")?;
let new_fname_tree = new_db.open_tree("filename")?;
debug!(
"Migrating files for {}",
String::from_utf8_lossy(alias.as_ref())
);
if let Some(filename) = old_db.self_tree().get(&hash)? {
main_tree.insert(&hash, filename.clone())?;
new_fname_tree.insert(filename, hash.clone())?;
main_tree.flush()?;
new_fname_tree.flush()?;
} else {
warn!("Missing filename");
}
let key = alias_key(&hash, id);
if let Some(v) = old_db.self_tree().get(&key)? {
main_tree.insert(key, v)?;
main_tree.flush()?;
} else {
warn!("Not migrating alias {} id {}", String::from_utf8_lossy(&alias), id);
return Ok(());
}
let (start, end) = variant_key_bounds(&hash);
if main_tree.range(start.clone()..end.clone()).next().is_none() {
let mut counter = 0;
for res in old_db.self_tree().range(start.clone()..end.clone()) {
counter += 1;
let (k, v) = res?;
debug!("Moving variant #{} for {}", counter, String::from_utf8_lossy(v.as_ref()));
main_tree.insert(k, v)?;
main_tree.flush()?;
}
debug!("Moved {} variants for {}", counter, String::from_utf8_lossy(alias.as_ref()));
}
Ok(())
}
pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) { pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
let mut start = hash.to_vec(); let mut start = hash.to_vec();
start.extend(&[0]); start.extend(&[0]);
@ -289,52 +129,3 @@ pub(crate) fn alias_key(hash: &[u8], id: &str) -> Vec<u8> {
key key
} }
const VALID: &[char] = &[
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
];
fn increment_alphanumeric(input: &str) -> String {
let (_, output) = input.chars().rev().fold((true, String::new()), |(incr_next, mut acc), item| {
if incr_next {
let mut index = None;
for (i, test) in VALID.iter().enumerate() {
if *test == item {
index = Some(i);
}
}
let index = index.unwrap_or(0);
let (set_incr_next, next_index) = if index == (VALID.len() - 1) {
(true, 0)
} else {
(false, index + 1)
};
acc.extend(&[VALID[next_index]]);
(set_incr_next, acc)
} else {
acc.extend(&[item]);
(false, acc)
}
});
output.chars().rev().collect()
}
#[cfg(test)]
mod tests {
use super::increment_alphanumeric;
#[test]
fn increments() {
assert_eq!(increment_alphanumeric("hello"), "hellp");
assert_eq!(increment_alphanumeric("0"), "1");
assert_eq!(increment_alphanumeric("9"), "A");
assert_eq!(increment_alphanumeric("Z"), "a");
assert_eq!(increment_alphanumeric("z"), "0");
assert_eq!(increment_alphanumeric("az"), "b0");
assert_eq!(increment_alphanumeric("19"), "1A");
assert_eq!(increment_alphanumeric("AZ"), "Aa");
}
}

85
src/migrate/s032.rs
View File

@ -1,85 +0,0 @@
use crate::{
migrate::{SledDb, SledIter, SledTree},
UploadError,
};
use std::path::PathBuf;
const SLED_032: &str = "db-0.32";
const SLED_0320_RC1: &str = "db";
pub(crate) fn exists_rc1(mut base: PathBuf) -> bool {
base.push(SLED_0320_RC1);
std::fs::metadata(base).is_ok()
}
pub(crate) fn open_rc1(mut base: PathBuf) -> Result<sled032::Db, UploadError> {
base.push(SLED_0320_RC1);
Ok(sled032::open(base)?)
}
pub(crate) fn exists(mut base: PathBuf) -> bool {
base.push("sled");
base.push(SLED_032);
std::fs::metadata(base).is_ok()
}
pub(crate) fn open(mut base: PathBuf) -> Result<sled032::Db, UploadError> {
base.push("sled");
base.push(SLED_032);
Ok(sled032::open(base)?)
}
impl SledDb for sled032::Db {
type SledTree = sled032::Tree;
fn open_tree(&self, name: &str) -> Result<Self::SledTree, UploadError> {
Ok(sled032::Db::open_tree(self, name)?)
}
fn self_tree(&self) -> &Self::SledTree {
&*self
}
}
impl SledTree for sled032::Tree {
fn get<K>(&self, key: K) -> Result<Option<Vec<u8>>, UploadError>
where
K: AsRef<[u8]>,
{
Ok(sled032::Tree::get(self, key)?.map(|v| Vec::from(v.as_ref())))
}
fn insert<K, V>(&self, key: K, value: V) -> Result<(), UploadError>
where
K: AsRef<[u8]>,
V: AsRef<[u8]>,
{
Ok(sled032::Tree::insert(self, key, value.as_ref().to_vec()).map(|_| ())?)
}
fn iter(&self) -> SledIter {
Box::new(sled032::Tree::iter(self).map(|res| {
res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec()))
.map_err(UploadError::from)
}))
}
fn range<K, R>(&self, range: R) -> SledIter
where
K: AsRef<[u8]>,
R: std::ops::RangeBounds<K>,
{
Box::new(sled032::Tree::range(self, range).map(|res| {
res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec()))
.map_err(UploadError::from)
}))
}
fn flush(&self) -> Result<(), UploadError> {
sled032::Tree::flush(self).map(|_| ()).map_err(UploadError::from)
}
}

58
src/processor.rs
View File

@ -113,6 +113,63 @@ impl Processor for Thumbnail {
} }
} }
pub(crate) struct Resize(usize);
impl Processor for Resize {
fn name() -> &'static str
where
Self: Sized,
{
"resize"
}
fn is_processor(s: &str) -> bool
where
Self: Sized,
{
s == Self::name()
}
fn parse(_: &str, v: &str) -> Option<Box<dyn Processor + Send>>
where
Self: Sized,
{
let size = v.parse().ok()?;
Some(Box::new(Resize(size)))
}
fn path(&self, mut path: PathBuf) -> PathBuf {
path.push(Self::name());
path.push(self.0.to_string());
path
}
fn process(&self, wand: &mut MagickWand) -> Result<(), UploadError> {
debug!("Resize");
let width = wand.get_image_width();
let height = wand.get_image_height();
if width > self.0 || height > self.0 {
let width_ratio = width as f64 / self.0 as f64;
let height_ratio = height as f64 / self.0 as f64;
let (new_width, new_height) = if width_ratio < height_ratio {
(width as f64 / height_ratio, self.0 as f64)
} else {
(self.0 as f64, height as f64 / width_ratio)
};
wand.resize_image(
new_width as usize,
new_height as usize,
magick_rust::bindings::FilterType_Lanczos2Filter,
);
}
Ok(())
}
}
pub(crate) struct Blur(f64); pub(crate) struct Blur(f64);
impl Processor for Blur { impl Processor for Blur {
@ -178,6 +235,7 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> ProcessChain {
parse!(Identity, k, v); parse!(Identity, k, v);
parse!(Thumbnail, k, v); parse!(Thumbnail, k, v);
parse!(Resize, k, v);
parse!(Blur, k, v); parse!(Blur, k, v);
debug!("Skipping {}: {}, invalid", k, v); debug!("Skipping {}: {}, invalid", k, v);

86
src/upload_manager.rs
View File

@ -1,7 +1,7 @@
use crate::{ use crate::{
config::Format, config::Format,
error::UploadError, error::UploadError,
migrate::{alias_key_bounds, variant_key_bounds, LatestDb, alias_id_key, alias_key}, migrate::{alias_id_key, alias_key, alias_key_bounds, variant_key_bounds, LatestDb},
to_ext, to_ext,
validate::validate_image, validate::validate_image,
}; };
@ -46,6 +46,23 @@ impl std::fmt::Debug for UploadManager {
type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, E>>>>; type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, E>>>>;
#[derive(serde::Deserialize, serde::Serialize)]
pub(crate) struct Details {
created_at: time::OffsetDateTime,
}
impl Details {
pub(crate) fn now() -> Self {
Details {
created_at: time::OffsetDateTime::now_utc(),
}
}
pub(crate) fn system_time(&self) -> std::time::SystemTime {
self.created_at.into()
}
}
struct FilenameIVec { struct FilenameIVec {
inner: sled::IVec, inner: sled::IVec,
} }
@ -149,6 +166,55 @@ impl UploadManager {
Ok(()) Ok(())
} }
/// Get the image details for a given variant
pub(crate) async fn variant_details(
&self,
path: PathBuf,
filename: String,
) -> Result<Option<Details>, UploadError> {
let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
let fname_tree = self.inner.filename_tree.clone();
debug!("Getting hash");
let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes()))
.await?
.ok_or(UploadError::MissingFilename)?;
let key = variant_details_key(&hash, &path_string);
let main_tree = self.inner.main_tree.clone();
debug!("Getting details");
let opt = match web::block(move || main_tree.get(key)).await? {
Some(ivec) => Some(serde_json::from_slice(&ivec)?),
None => None,
};
debug!("Got details");
Ok(opt)
}
pub(crate) async fn store_variant_details(
&self,
path: PathBuf,
filename: String,
) -> Result<(), UploadError> {
let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
let fname_tree = self.inner.filename_tree.clone();
debug!("Getting hash");
let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes()))
.await?
.ok_or(UploadError::MissingFilename)?;
let key = variant_details_key(&hash, &path_string);
let main_tree = self.inner.main_tree.clone();
let details_value = serde_json::to_string(&Details::now())?;
debug!("Storing details");
web::block(move || main_tree.insert(key, details_value.as_bytes())).await?;
debug!("Stored details");
Ok(())
}
/// Get a list of aliases for a given file /// Get a list of aliases for a given file
pub(crate) async fn aliases_by_filename( pub(crate) async fn aliases_by_filename(
&self, &self,
@ -470,9 +536,13 @@ impl UploadManager {
for key in keys { for key in keys {
let main_tree = self.inner.main_tree.clone(); let main_tree = self.inner.main_tree.clone();
if let Some(path) = web::block(move || main_tree.remove(key)).await? { if let Some(path) = web::block(move || main_tree.remove(key)).await? {
debug!("Deleting {:?}", String::from_utf8(path.to_vec())); let s = String::from_utf8_lossy(&path);
if let Err(e) = remove_path(path).await { debug!("Deleting {}", s);
errors.push(e); // ignore json objects
if !s.starts_with('{') {
if let Err(e) = remove_path(path).await {
errors.push(e);
}
} }
} }
} }
@ -762,3 +832,11 @@ fn variant_key(hash: &[u8], path: &str) -> Vec<u8> {
key.extend(path.as_bytes()); key.extend(path.as_bytes());
key key
} }
fn variant_details_key(hash: &[u8], path: &str) -> Vec<u8> {
let mut key = hash.to_vec();
key.extend(&[2]);
key.extend(path.as_bytes());
key.extend(b"details");
key
}