mirror of
https://git.asonix.dog/asonix/pict-rs
synced 2024-12-22 11:21:24 +00:00
Add initial details implementation
Details is an object generated for each image, containing it's created_at date to properly set response headers. It also can be expanding in the future to include an image's dimensions, which we can expose via an API endpoint. In addition to this, Details allows us the ability to re-generate processed images. For now, all images which do not yet have an associated Details will be re-generated upon access. Details is set for each thumbnail on generation, so this will only happen once for each image.
This commit is contained in:
parent
e3dbd5e791
commit
0bf7e0f688
9 changed files with 224 additions and 440 deletions
136
Cargo.lock
generated
136
Cargo.lock
generated
|
|
@ -361,15 +361,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.34"
|
||||
version = "1.0.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7"
|
||||
|
||||
[[package]]
|
||||
name = "array-init"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f30bbe2f5e3d117f55bd8c7a1f9191e4a5deba9f15f595bbea4f670c59c765db"
|
||||
checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4"
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
|
|
@ -564,9 +558,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.65"
|
||||
version = "1.0.66"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15"
|
||||
checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
|
|
@ -628,20 +622,11 @@ dependencies = [
|
|||
"vec_map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cloudabi"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const_fn"
|
||||
version = "0.4.3"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab"
|
||||
checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826"
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
|
|
@ -675,21 +660,6 @@ dependencies = [
|
|||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if 0.1.10",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"lazy_static",
|
||||
"maybe-uninit",
|
||||
"memoffset 0.5.6",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.1"
|
||||
|
|
@ -698,23 +668,12 @@ checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
|
|||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"const_fn",
|
||||
"crossbeam-utils 0.8.1",
|
||||
"crossbeam-utils",
|
||||
"lazy_static",
|
||||
"memoffset 0.6.1",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if 0.1.10",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.1"
|
||||
|
|
@ -1208,9 +1167,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.80"
|
||||
version = "0.2.81"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
|
||||
checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
|
|
@ -1299,27 +1258,12 @@ version = "0.1.8"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
|
||||
|
||||
[[package]]
|
||||
name = "maybe-uninit"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.6.1"
|
||||
|
|
@ -1389,9 +1333,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "net2"
|
||||
version = "0.2.36"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02"
|
||||
checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"libc",
|
||||
|
|
@ -1479,12 +1423,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
|
||||
checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"cloudabi",
|
||||
"cfg-if 1.0.0",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
|
|
@ -1506,7 +1449,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
|||
|
||||
[[package]]
|
||||
name = "pict-rs"
|
||||
version = "0.2.6"
|
||||
version = "0.3.0-alpha.0"
|
||||
dependencies = [
|
||||
"actix-form-data",
|
||||
"actix-fs",
|
||||
|
|
@ -1526,10 +1469,10 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"sled 0.32.1",
|
||||
"sled 0.34.6",
|
||||
"sled",
|
||||
"structopt",
|
||||
"thiserror",
|
||||
"time 0.2.23",
|
||||
"tracing",
|
||||
"tracing-futures",
|
||||
"tracing-subscriber",
|
||||
|
|
@ -1856,18 +1799,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
|||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.117"
|
||||
version = "1.0.118"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a"
|
||||
checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.117"
|
||||
version = "1.0.118"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e"
|
||||
checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1960,24 +1903,6 @@ version = "0.4.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
|
||||
|
||||
[[package]]
|
||||
name = "sled"
|
||||
version = "0.32.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e3dbbb8ee10611bd1d020767c27599ccbbf8365f7e0ed7e54429cc8b9433ad8"
|
||||
dependencies = [
|
||||
"array-init",
|
||||
"backtrace",
|
||||
"crc32fast",
|
||||
"crossbeam-epoch 0.8.2",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"fs2",
|
||||
"fxhash",
|
||||
"libc",
|
||||
"log",
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sled"
|
||||
version = "0.34.6"
|
||||
|
|
@ -1985,8 +1910,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"crossbeam-epoch 0.9.1",
|
||||
"crossbeam-utils 0.8.1",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"fs2",
|
||||
"fxhash",
|
||||
"libc",
|
||||
|
|
@ -1996,9 +1921,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.5.0"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85"
|
||||
checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
|
|
@ -2108,9 +2033,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.53"
|
||||
version = "1.0.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68"
|
||||
checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2192,6 +2117,7 @@ checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b"
|
|||
dependencies = [
|
||||
"const_fn",
|
||||
"libc",
|
||||
"serde",
|
||||
"standback",
|
||||
"stdweb",
|
||||
"time-macros",
|
||||
|
|
@ -2239,9 +2165,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
|||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "0.2.23"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff"
|
||||
checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "pict-rs"
|
||||
description = "A simple image hosting service"
|
||||
version = "0.2.6"
|
||||
version = "0.3.0-alpha.0"
|
||||
authors = ["asonix <asonix@asonix.dog>"]
|
||||
license = "AGPL-3.0"
|
||||
readme = "README.md"
|
||||
|
|
@ -27,10 +27,10 @@ rexiv2 = "0.9.1"
|
|||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
sha2 = "0.9.0"
|
||||
sled032 = { version = "0.32.0", package = "sled" }
|
||||
sled = { version = "0.34.4" }
|
||||
structopt = "0.3.14"
|
||||
thiserror = "1.0"
|
||||
time = { version = "0.2.23", features = ["serde"] }
|
||||
tracing = "0.1.15"
|
||||
tracing-futures = "0.2.4"
|
||||
tracing-subscriber = { version = "0.2.5", features = ["fmt", "tracing-log"] }
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ _a simple image hosting service_
|
|||
## Usage
|
||||
### Running
|
||||
```
|
||||
pict-rs 0.2.0-alpha.3
|
||||
pict-rs 0.3.0-alpha.0
|
||||
|
||||
USAGE:
|
||||
pict-rs [FLAGS] [OPTIONS] --path <path>
|
||||
|
|
@ -109,7 +109,10 @@ pict-rs offers the following endpoints:
|
|||
existing transformations include
|
||||
- `identity=true`: apply no changes
|
||||
- `blur={float}`: apply a gaussian blur to the file
|
||||
- `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square
|
||||
- `thumbnail={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}`
|
||||
square using raw pixel sampling
|
||||
- `resize={int}`: produce a thumbnail of the image fitting inside an `{int}` by `{int}` square
|
||||
using a Lanczos2 filter. This is slower than sampling but looks a bit better in some cases
|
||||
|
||||
Supported `ext` file extensions include `png`, `jpg`, and `webp`
|
||||
|
||||
|
|
|
|||
|
|
@ -9,9 +9,6 @@ pub(crate) enum UploadError {
|
|||
#[error("Couldn't save file, {0}")]
|
||||
Save(#[from] actix_fs::Error),
|
||||
|
||||
#[error("Error in DB migration, {0}")]
|
||||
Migrate(#[from] sled032::Error),
|
||||
|
||||
#[error("Error in DB, {0}")]
|
||||
Db(#[from] sled::Error),
|
||||
|
||||
|
|
@ -71,6 +68,9 @@ pub(crate) enum UploadError {
|
|||
|
||||
#[error("Error in MagickWand, {0}")]
|
||||
Wand(String),
|
||||
|
||||
#[error("{0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
impl From<actix_web::client::SendRequestError> for UploadError {
|
||||
|
|
|
|||
71
src/main.rs
71
src/main.rs
|
|
@ -26,7 +26,7 @@ use self::{
|
|||
error::UploadError,
|
||||
middleware::{Internal, Tracing},
|
||||
processor::process_image,
|
||||
upload_manager::UploadManager,
|
||||
upload_manager::{Details, UploadManager},
|
||||
validate::{image_webp, video_mp4},
|
||||
};
|
||||
|
||||
|
|
@ -289,38 +289,28 @@ async fn process(
|
|||
.map_err(|_| UploadError::UnsupportedFormat)?;
|
||||
let processed_name = format!("{}.{}", name, ext);
|
||||
let base = manager.image_dir();
|
||||
let mut path = self::processor::build_path(base, &chain, processed_name);
|
||||
|
||||
if let Some((updated_path, exists)) = self::processor::prepare_image(path.clone()).await? {
|
||||
path = updated_path.clone();
|
||||
|
||||
if exists.is_new() {
|
||||
// Save the transcoded file in another task
|
||||
debug!("Spawning storage task");
|
||||
let span = Span::current();
|
||||
let manager2 = manager.clone();
|
||||
let name = name.clone();
|
||||
actix_rt::spawn(async move {
|
||||
let entered = span.enter();
|
||||
if let Err(e) = manager2.store_variant(updated_path, name).await {
|
||||
error!("Error storing variant, {}", e);
|
||||
return;
|
||||
}
|
||||
drop(entered);
|
||||
});
|
||||
}
|
||||
}
|
||||
let thumbnail_path = self::processor::build_path(base, &chain, processed_name);
|
||||
|
||||
// If the thumbnail doesn't exist, we need to create it
|
||||
if let Err(e) = actix_fs::metadata(path.clone()).await {
|
||||
let thumbnail_exists = if let Err(e) = actix_fs::metadata(thumbnail_path.clone()).await {
|
||||
if e.kind() != Some(std::io::ErrorKind::NotFound) {
|
||||
error!("Error looking up processed image, {}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
false
|
||||
} else {
|
||||
true
|
||||
};
|
||||
|
||||
let details = manager
|
||||
.variant_details(thumbnail_path.clone(), name.clone())
|
||||
.await?;
|
||||
|
||||
if !thumbnail_exists || details.is_none() {
|
||||
let mut original_path = manager.image_dir();
|
||||
original_path.push(name.clone());
|
||||
|
||||
// Create and save a JPG for motion images (gif, mp4)
|
||||
if let Some((updated_path, exists)) =
|
||||
self::processor::prepare_image(original_path.clone()).await?
|
||||
{
|
||||
|
|
@ -346,14 +336,25 @@ async fn process(
|
|||
// apply chain to the provided image
|
||||
let img_bytes = process_image(original_path.clone(), chain, format).await?;
|
||||
|
||||
let path2 = path.clone();
|
||||
let path2 = thumbnail_path.clone();
|
||||
let img_bytes2 = img_bytes.clone();
|
||||
|
||||
// Save the file in another task, we want to return the thumbnail now
|
||||
debug!("Spawning storage task");
|
||||
let span = Span::current();
|
||||
let store_details = details.is_none();
|
||||
actix_rt::spawn(async move {
|
||||
let entered = span.enter();
|
||||
if store_details {
|
||||
debug!("Storing details");
|
||||
if let Err(e) = manager
|
||||
.store_variant_details(path2.clone(), name.clone())
|
||||
.await
|
||||
{
|
||||
error!("Error storing details, {}", e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if let Err(e) = manager.store_variant(path2.clone(), name).await {
|
||||
error!("Error storing variant, {}", e);
|
||||
return;
|
||||
|
|
@ -365,23 +366,27 @@ async fn process(
|
|||
drop(entered);
|
||||
});
|
||||
|
||||
let details = details.unwrap_or(Details::now());
|
||||
|
||||
return Ok(srv_response(
|
||||
Box::pin(futures::stream::once(async {
|
||||
Ok(img_bytes) as Result<_, UploadError>
|
||||
})),
|
||||
content_type,
|
||||
7 * DAYS,
|
||||
SystemTime::now(),
|
||||
details.system_time(),
|
||||
));
|
||||
}
|
||||
|
||||
let stream = actix_fs::read_to_stream(path).await?;
|
||||
let stream = actix_fs::read_to_stream(thumbnail_path).await?;
|
||||
|
||||
let details = details.unwrap_or(Details::now());
|
||||
|
||||
Ok(srv_response(
|
||||
stream,
|
||||
content_type,
|
||||
7 * DAYS,
|
||||
SystemTime::now(),
|
||||
details.system_time(),
|
||||
))
|
||||
}
|
||||
|
||||
|
|
@ -394,7 +399,15 @@ async fn serve(
|
|||
let name = manager.from_alias(alias.into_inner()).await?;
|
||||
let content_type = from_name(&name)?;
|
||||
let mut path = manager.image_dir();
|
||||
path.push(name);
|
||||
path.push(name.clone());
|
||||
|
||||
let details = manager.variant_details(path.clone(), name.clone()).await?;
|
||||
|
||||
if details.is_none() {
|
||||
manager.store_variant_details(path.clone(), name).await?;
|
||||
}
|
||||
|
||||
let details = details.unwrap_or(Details::now());
|
||||
|
||||
let stream = actix_fs::read_to_stream(path).await?;
|
||||
|
||||
|
|
@ -402,7 +415,7 @@ async fn serve(
|
|||
stream,
|
||||
content_type,
|
||||
7 * DAYS,
|
||||
SystemTime::now(),
|
||||
details.system_time(),
|
||||
))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
use crate::UploadError;
|
||||
use sled;
|
||||
use std::path::PathBuf;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
mod s032;
|
||||
mod s034;
|
||||
|
||||
type SledIter = Box<dyn Iterator<Item = Result<(Vec<u8>, Vec<u8>), UploadError>>>;
|
||||
|
|
@ -68,9 +66,7 @@ impl LatestDb {
|
|||
|
||||
loop {
|
||||
let root_dir2 = root_dir.clone();
|
||||
let res = std::panic::catch_unwind(move || {
|
||||
version.migrate(root_dir2)
|
||||
});
|
||||
let res = std::panic::catch_unwind(move || version.migrate(root_dir2));
|
||||
|
||||
if let Ok(res) = res {
|
||||
return res;
|
||||
|
|
@ -81,8 +77,6 @@ impl LatestDb {
|
|||
|
||||
#[derive(Clone, Copy)]
|
||||
enum DbVersion {
|
||||
Sled0320Rc1,
|
||||
Sled032,
|
||||
Sled034,
|
||||
Fresh,
|
||||
}
|
||||
|
|
@ -93,170 +87,16 @@ impl DbVersion {
|
|||
return DbVersion::Sled034;
|
||||
}
|
||||
|
||||
if s032::exists(root.clone()) {
|
||||
return DbVersion::Sled032;
|
||||
}
|
||||
|
||||
if s032::exists_rc1(root.clone()) {
|
||||
return DbVersion::Sled0320Rc1;
|
||||
}
|
||||
|
||||
DbVersion::Fresh
|
||||
}
|
||||
|
||||
fn migrate(self, root: PathBuf) -> Result<sled::Db, UploadError> {
|
||||
match self {
|
||||
DbVersion::Sled0320Rc1 => migrate_0_32_0_rc1(root),
|
||||
DbVersion::Sled032 => migrate_0_32(root),
|
||||
DbVersion::Sled034 | DbVersion::Fresh => s034::open(root),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn migrate_0_32_0_rc1(root: PathBuf) -> Result<sled::Db, UploadError> {
|
||||
info!("Migrating database from 0.32.0-rc1 to 0.34.0");
|
||||
|
||||
let old_db = s032::open_rc1(root.clone())?;
|
||||
let new_db = s034::open(root)?;
|
||||
|
||||
migrate(old_db, new_db)
|
||||
}
|
||||
|
||||
fn migrate_0_32(root: PathBuf) -> Result<sled::Db, UploadError> {
|
||||
info!("Migrating database from 0.32 to 0.34");
|
||||
|
||||
let old_db = s032::open(root.clone())?;
|
||||
let new_db = s034::open(root)?;
|
||||
|
||||
migrate(old_db, new_db)
|
||||
}
|
||||
|
||||
fn migrate<Old, New>(old_db: Old, new_db: New) -> Result<New, UploadError>
|
||||
where
|
||||
Old: SledDb,
|
||||
New: SledDb,
|
||||
{
|
||||
let old_alias_tree = old_db.open_tree("alias")?;
|
||||
let new_alias_tree = new_db.open_tree("alias")?;
|
||||
|
||||
let new_migrate_tree = new_db.open_tree("migrate")?;
|
||||
if let Some(_) = new_migrate_tree.get("done")? {
|
||||
return Ok(new_db);
|
||||
}
|
||||
|
||||
let (iterator, mut counter) = if let Some(last_migrated) = new_migrate_tree.get("last_migrated")? {
|
||||
let mut last_migrated = String::from_utf8_lossy(&last_migrated).to_string();
|
||||
info!("Previous migration failed after {}, attempting to skip", last_migrated);
|
||||
if let Some(index) = last_migrated.find('.') {
|
||||
last_migrated = last_migrated.split_at(index).0.to_owned();
|
||||
}
|
||||
if last_migrated.len() > 3 {
|
||||
last_migrated = last_migrated.split_at(3).0.to_owned();
|
||||
}
|
||||
let last_migrated = increment_alphanumeric(&last_migrated).as_bytes().to_owned();
|
||||
new_migrate_tree.insert("last_migrated", last_migrated.clone())?;
|
||||
new_migrate_tree.flush()?;
|
||||
if let Some(count) = new_migrate_tree.get("counter")? {
|
||||
(old_alias_tree.range(last_migrated..), String::from_utf8_lossy(&count).parse::<usize>().unwrap())
|
||||
} else {
|
||||
(old_alias_tree.range(last_migrated..), 0)
|
||||
}
|
||||
} else {
|
||||
(old_alias_tree.iter(), 0)
|
||||
};
|
||||
|
||||
for res in iterator {
|
||||
let (k, _) = res?;
|
||||
|
||||
if let Some(v) = old_alias_tree.get(&k)? {
|
||||
if !k.contains(&b"/"[0]) {
|
||||
if let Some(id) = old_alias_tree.get(alias_id_key(&String::from_utf8_lossy(&k)))? {
|
||||
counter += 1;
|
||||
debug!("Migrating alias #{}", counter);
|
||||
// k is an alias
|
||||
migrate_main_tree(&k, &v, &old_db, &new_db, &String::from_utf8_lossy(&id))?;
|
||||
debug!(
|
||||
"Moving alias -> hash for alias {}",
|
||||
String::from_utf8_lossy(k.as_ref()),
|
||||
);
|
||||
new_migrate_tree.insert("counter", format!("{}", counter))?;
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
"Moving {}, {}",
|
||||
String::from_utf8_lossy(k.as_ref()),
|
||||
String::from_utf8_lossy(v.as_ref())
|
||||
);
|
||||
}
|
||||
new_alias_tree.insert(k.clone(), v)?;
|
||||
new_migrate_tree.insert("last_migrated", k)?;
|
||||
new_alias_tree.flush()?;
|
||||
new_migrate_tree.flush()?;
|
||||
} else {
|
||||
warn!("MISSING {}", String::from_utf8_lossy(k.as_ref()));
|
||||
}
|
||||
}
|
||||
info!("Moved {} unique aliases", counter);
|
||||
|
||||
new_migrate_tree.insert("done", "true")?;
|
||||
new_migrate_tree.flush()?;
|
||||
|
||||
Ok(new_db)
|
||||
}
|
||||
|
||||
fn migrate_main_tree<Old, New>(
|
||||
alias: &[u8],
|
||||
hash: &[u8],
|
||||
old_db: Old,
|
||||
new_db: New,
|
||||
id: &str,
|
||||
) -> Result<(), UploadError>
|
||||
where
|
||||
Old: SledDb,
|
||||
New: SledDb,
|
||||
{
|
||||
let main_tree = new_db.open_tree("main")?;
|
||||
|
||||
let new_fname_tree = new_db.open_tree("filename")?;
|
||||
|
||||
debug!(
|
||||
"Migrating files for {}",
|
||||
String::from_utf8_lossy(alias.as_ref())
|
||||
);
|
||||
if let Some(filename) = old_db.self_tree().get(&hash)? {
|
||||
main_tree.insert(&hash, filename.clone())?;
|
||||
new_fname_tree.insert(filename, hash.clone())?;
|
||||
main_tree.flush()?;
|
||||
new_fname_tree.flush()?;
|
||||
} else {
|
||||
warn!("Missing filename");
|
||||
}
|
||||
|
||||
let key = alias_key(&hash, id);
|
||||
if let Some(v) = old_db.self_tree().get(&key)? {
|
||||
main_tree.insert(key, v)?;
|
||||
main_tree.flush()?;
|
||||
} else {
|
||||
warn!("Not migrating alias {} id {}", String::from_utf8_lossy(&alias), id);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (start, end) = variant_key_bounds(&hash);
|
||||
if main_tree.range(start.clone()..end.clone()).next().is_none() {
|
||||
let mut counter = 0;
|
||||
for res in old_db.self_tree().range(start.clone()..end.clone()) {
|
||||
counter += 1;
|
||||
let (k, v) = res?;
|
||||
debug!("Moving variant #{} for {}", counter, String::from_utf8_lossy(v.as_ref()));
|
||||
main_tree.insert(k, v)?;
|
||||
main_tree.flush()?;
|
||||
}
|
||||
debug!("Moved {} variants for {}", counter, String::from_utf8_lossy(alias.as_ref()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut start = hash.to_vec();
|
||||
start.extend(&[0]);
|
||||
|
|
@ -289,52 +129,3 @@ pub(crate) fn alias_key(hash: &[u8], id: &str) -> Vec<u8> {
|
|||
|
||||
key
|
||||
}
|
||||
|
||||
const VALID: &[char] = &[
|
||||
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
||||
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
|
||||
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
|
||||
];
|
||||
|
||||
fn increment_alphanumeric(input: &str) -> String {
|
||||
let (_, output) = input.chars().rev().fold((true, String::new()), |(incr_next, mut acc), item| {
|
||||
if incr_next {
|
||||
let mut index = None;
|
||||
for (i, test) in VALID.iter().enumerate() {
|
||||
if *test == item {
|
||||
index = Some(i);
|
||||
}
|
||||
}
|
||||
let index = index.unwrap_or(0);
|
||||
let (set_incr_next, next_index) = if index == (VALID.len() - 1) {
|
||||
(true, 0)
|
||||
} else {
|
||||
(false, index + 1)
|
||||
};
|
||||
acc.extend(&[VALID[next_index]]);
|
||||
(set_incr_next, acc)
|
||||
} else {
|
||||
acc.extend(&[item]);
|
||||
(false, acc)
|
||||
}
|
||||
});
|
||||
|
||||
output.chars().rev().collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::increment_alphanumeric;
|
||||
|
||||
#[test]
|
||||
fn increments() {
|
||||
assert_eq!(increment_alphanumeric("hello"), "hellp");
|
||||
assert_eq!(increment_alphanumeric("0"), "1");
|
||||
assert_eq!(increment_alphanumeric("9"), "A");
|
||||
assert_eq!(increment_alphanumeric("Z"), "a");
|
||||
assert_eq!(increment_alphanumeric("z"), "0");
|
||||
assert_eq!(increment_alphanumeric("az"), "b0");
|
||||
assert_eq!(increment_alphanumeric("19"), "1A");
|
||||
assert_eq!(increment_alphanumeric("AZ"), "Aa");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,85 +0,0 @@
|
|||
use crate::{
|
||||
migrate::{SledDb, SledIter, SledTree},
|
||||
UploadError,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
|
||||
const SLED_032: &str = "db-0.32";
|
||||
const SLED_0320_RC1: &str = "db";
|
||||
|
||||
pub(crate) fn exists_rc1(mut base: PathBuf) -> bool {
|
||||
base.push(SLED_0320_RC1);
|
||||
|
||||
std::fs::metadata(base).is_ok()
|
||||
}
|
||||
|
||||
pub(crate) fn open_rc1(mut base: PathBuf) -> Result<sled032::Db, UploadError> {
|
||||
base.push(SLED_0320_RC1);
|
||||
|
||||
Ok(sled032::open(base)?)
|
||||
}
|
||||
|
||||
pub(crate) fn exists(mut base: PathBuf) -> bool {
|
||||
base.push("sled");
|
||||
base.push(SLED_032);
|
||||
|
||||
std::fs::metadata(base).is_ok()
|
||||
}
|
||||
|
||||
pub(crate) fn open(mut base: PathBuf) -> Result<sled032::Db, UploadError> {
|
||||
base.push("sled");
|
||||
base.push(SLED_032);
|
||||
|
||||
Ok(sled032::open(base)?)
|
||||
}
|
||||
|
||||
impl SledDb for sled032::Db {
|
||||
type SledTree = sled032::Tree;
|
||||
|
||||
fn open_tree(&self, name: &str) -> Result<Self::SledTree, UploadError> {
|
||||
Ok(sled032::Db::open_tree(self, name)?)
|
||||
}
|
||||
|
||||
fn self_tree(&self) -> &Self::SledTree {
|
||||
&*self
|
||||
}
|
||||
}
|
||||
|
||||
impl SledTree for sled032::Tree {
|
||||
fn get<K>(&self, key: K) -> Result<Option<Vec<u8>>, UploadError>
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
{
|
||||
Ok(sled032::Tree::get(self, key)?.map(|v| Vec::from(v.as_ref())))
|
||||
}
|
||||
|
||||
fn insert<K, V>(&self, key: K, value: V) -> Result<(), UploadError>
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: AsRef<[u8]>,
|
||||
{
|
||||
Ok(sled032::Tree::insert(self, key, value.as_ref().to_vec()).map(|_| ())?)
|
||||
}
|
||||
|
||||
fn iter(&self) -> SledIter {
|
||||
Box::new(sled032::Tree::iter(self).map(|res| {
|
||||
res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec()))
|
||||
.map_err(UploadError::from)
|
||||
}))
|
||||
}
|
||||
|
||||
fn range<K, R>(&self, range: R) -> SledIter
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
R: std::ops::RangeBounds<K>,
|
||||
{
|
||||
Box::new(sled032::Tree::range(self, range).map(|res| {
|
||||
res.map(|(k, v)| (k.as_ref().to_vec(), v.as_ref().to_vec()))
|
||||
.map_err(UploadError::from)
|
||||
}))
|
||||
}
|
||||
|
||||
fn flush(&self) -> Result<(), UploadError> {
|
||||
sled032::Tree::flush(self).map(|_| ()).map_err(UploadError::from)
|
||||
}
|
||||
}
|
||||
|
|
@ -113,6 +113,63 @@ impl Processor for Thumbnail {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Resize(usize);
|
||||
|
||||
impl Processor for Resize {
|
||||
fn name() -> &'static str
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
"resize"
|
||||
}
|
||||
|
||||
fn is_processor(s: &str) -> bool
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
s == Self::name()
|
||||
}
|
||||
|
||||
fn parse(_: &str, v: &str) -> Option<Box<dyn Processor + Send>>
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
let size = v.parse().ok()?;
|
||||
Some(Box::new(Resize(size)))
|
||||
}
|
||||
|
||||
fn path(&self, mut path: PathBuf) -> PathBuf {
|
||||
path.push(Self::name());
|
||||
path.push(self.0.to_string());
|
||||
path
|
||||
}
|
||||
|
||||
fn process(&self, wand: &mut MagickWand) -> Result<(), UploadError> {
|
||||
debug!("Resize");
|
||||
let width = wand.get_image_width();
|
||||
let height = wand.get_image_height();
|
||||
|
||||
if width > self.0 || height > self.0 {
|
||||
let width_ratio = width as f64 / self.0 as f64;
|
||||
let height_ratio = height as f64 / self.0 as f64;
|
||||
|
||||
let (new_width, new_height) = if width_ratio < height_ratio {
|
||||
(width as f64 / height_ratio, self.0 as f64)
|
||||
} else {
|
||||
(self.0 as f64, height as f64 / width_ratio)
|
||||
};
|
||||
|
||||
wand.resize_image(
|
||||
new_width as usize,
|
||||
new_height as usize,
|
||||
magick_rust::bindings::FilterType_Lanczos2Filter,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Blur(f64);
|
||||
|
||||
impl Processor for Blur {
|
||||
|
|
@ -178,6 +235,7 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> ProcessChain {
|
|||
|
||||
parse!(Identity, k, v);
|
||||
parse!(Thumbnail, k, v);
|
||||
parse!(Resize, k, v);
|
||||
parse!(Blur, k, v);
|
||||
|
||||
debug!("Skipping {}: {}, invalid", k, v);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use crate::{
|
||||
config::Format,
|
||||
error::UploadError,
|
||||
migrate::{alias_key_bounds, variant_key_bounds, LatestDb, alias_id_key, alias_key},
|
||||
migrate::{alias_id_key, alias_key, alias_key_bounds, variant_key_bounds, LatestDb},
|
||||
to_ext,
|
||||
validate::validate_image,
|
||||
};
|
||||
|
|
@ -46,6 +46,23 @@ impl std::fmt::Debug for UploadManager {
|
|||
|
||||
type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, E>>>>;
|
||||
|
||||
#[derive(serde::Deserialize, serde::Serialize)]
|
||||
pub(crate) struct Details {
|
||||
created_at: time::OffsetDateTime,
|
||||
}
|
||||
|
||||
impl Details {
|
||||
pub(crate) fn now() -> Self {
|
||||
Details {
|
||||
created_at: time::OffsetDateTime::now_utc(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn system_time(&self) -> std::time::SystemTime {
|
||||
self.created_at.into()
|
||||
}
|
||||
}
|
||||
|
||||
struct FilenameIVec {
|
||||
inner: sled::IVec,
|
||||
}
|
||||
|
|
@ -149,6 +166,55 @@ impl UploadManager {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the image details for a given variant
|
||||
pub(crate) async fn variant_details(
|
||||
&self,
|
||||
path: PathBuf,
|
||||
filename: String,
|
||||
) -> Result<Option<Details>, UploadError> {
|
||||
let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
|
||||
|
||||
let fname_tree = self.inner.filename_tree.clone();
|
||||
debug!("Getting hash");
|
||||
let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes()))
|
||||
.await?
|
||||
.ok_or(UploadError::MissingFilename)?;
|
||||
|
||||
let key = variant_details_key(&hash, &path_string);
|
||||
let main_tree = self.inner.main_tree.clone();
|
||||
debug!("Getting details");
|
||||
let opt = match web::block(move || main_tree.get(key)).await? {
|
||||
Some(ivec) => Some(serde_json::from_slice(&ivec)?),
|
||||
None => None,
|
||||
};
|
||||
debug!("Got details");
|
||||
|
||||
Ok(opt)
|
||||
}
|
||||
|
||||
pub(crate) async fn store_variant_details(
|
||||
&self,
|
||||
path: PathBuf,
|
||||
filename: String,
|
||||
) -> Result<(), UploadError> {
|
||||
let path_string = path.to_str().ok_or(UploadError::Path)?.to_string();
|
||||
|
||||
let fname_tree = self.inner.filename_tree.clone();
|
||||
debug!("Getting hash");
|
||||
let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes()))
|
||||
.await?
|
||||
.ok_or(UploadError::MissingFilename)?;
|
||||
|
||||
let key = variant_details_key(&hash, &path_string);
|
||||
let main_tree = self.inner.main_tree.clone();
|
||||
let details_value = serde_json::to_string(&Details::now())?;
|
||||
debug!("Storing details");
|
||||
web::block(move || main_tree.insert(key, details_value.as_bytes())).await?;
|
||||
debug!("Stored details");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get a list of aliases for a given file
|
||||
pub(crate) async fn aliases_by_filename(
|
||||
&self,
|
||||
|
|
@ -470,9 +536,13 @@ impl UploadManager {
|
|||
for key in keys {
|
||||
let main_tree = self.inner.main_tree.clone();
|
||||
if let Some(path) = web::block(move || main_tree.remove(key)).await? {
|
||||
debug!("Deleting {:?}", String::from_utf8(path.to_vec()));
|
||||
if let Err(e) = remove_path(path).await {
|
||||
errors.push(e);
|
||||
let s = String::from_utf8_lossy(&path);
|
||||
debug!("Deleting {}", s);
|
||||
// ignore json objects
|
||||
if !s.starts_with('{') {
|
||||
if let Err(e) = remove_path(path).await {
|
||||
errors.push(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -762,3 +832,11 @@ fn variant_key(hash: &[u8], path: &str) -> Vec<u8> {
|
|||
key.extend(path.as_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn variant_details_key(hash: &[u8], path: &str) -> Vec<u8> {
|
||||
let mut key = hash.to_vec();
|
||||
key.extend(&[2]);
|
||||
key.extend(path.as_bytes());
|
||||
key.extend(b"details");
|
||||
key
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue