LemmyNet/pict-rs
2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-22 19:31:35 +00:00
Code Releases Activity

Merge pull request 'Add blurhash support' (#54) from asonix/blurhash into main

Browse source 
Reviewed-on: https://git.asonix.dog/asonix/pict-rs/pulls/54
This commit is contained in:
asonix 2024-02-25 01:45:29 +00:00
commit 029beef61a
18 changed files with 600 additions and 586 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

308
Cargo.lock generated
View file

@ -131,7 +131,7 @@ dependencies = [
"futures-core", "futures-core",
"futures-util", "futures-util",
"mio", "mio",
"socket2 0.5.5", "socket2 0.5.6",
"tokio", "tokio",
"tokio-uring", "tokio-uring",
"tracing", "tracing",
@ -210,7 +210,7 @@ dependencies = [
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"smallvec", "smallvec",
"socket2 0.5.5", "socket2 0.5.6",
"time", "time",
"url", "url",
] ]
@ -269,9 +269,9 @@ dependencies = [
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.6.11" version = "0.6.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" checksum = "96b09b5178381e0874812a9b157f7fe84982617e48f71f4e3235482775e5b540"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"anstyle-parse", "anstyle-parse",
@ -317,9 +317,9 @@ dependencies = [
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.79" version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1"
[[package]] [[package]]
name = "async-stream" name = "async-stream"
@ -340,7 +340,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -351,7 +351,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -478,11 +478,16 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "blurhash-update"
version = "0.1.0"
source = "git+https://git.asonix.dog/asonix/blurhash-update#82f94e7f762ad8594b18ea9b238b5b276ade4afc"
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.14.0" version = "3.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
@ -507,12 +512,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.83" version = "1.0.87"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" checksum = "3286b845d0fccbdd15af433f61c5970e711987036cb468f437ff6badd70f4e24"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
@ -522,21 +524,21 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]] [[package]]
name = "chrono" name = "chrono"
version = "0.4.33" version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b"
dependencies = [ dependencies = [
"android-tzdata", "android-tzdata",
"iana-time-zone", "iana-time-zone",
"num-traits", "num-traits",
"windows-targets 0.52.0", "windows-targets 0.52.3",
] ]
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.4.18" version = "4.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@ -544,9 +546,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.4.18" version = "4.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb"
dependencies = [ dependencies = [
"anstream", "anstream",
"anstyle", "anstyle",
@ -556,21 +558,21 @@ dependencies = [
[[package]] [[package]]
name = "clap_derive" name = "clap_derive"
version = "4.4.7" version = "4.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47"
dependencies = [ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
name = "clap_lex" name = "clap_lex"
version = "0.6.0" version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
[[package]] [[package]]
name = "color-eyre" name = "color-eyre"
@ -697,9 +699,9 @@ dependencies = [
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.3.2" version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
] ]
@ -845,7 +847,7 @@ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -857,7 +859,7 @@ dependencies = [
"diesel_table_macro_syntax", "diesel_table_macro_syntax",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -866,7 +868,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
dependencies = [ dependencies = [
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -882,9 +884,9 @@ dependencies = [
[[package]] [[package]]
name = "either" name = "either"
version = "1.9.0" version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
[[package]] [[package]]
name = "encoding_rs" name = "encoding_rs"
@ -1025,7 +1027,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -1113,7 +1115,7 @@ dependencies = [
"futures-sink", "futures-sink",
"futures-util", "futures-util",
"http", "http",
"indexmap 2.2.2", "indexmap 2.2.3",
"slab", "slab",
"tokio", "tokio",
"tokio-util", "tokio-util",
@ -1126,20 +1128,14 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038"
dependencies = [
"ahash",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.3" version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
dependencies = [
"ahash",
]
[[package]] [[package]]
name = "hdrhistogram" name = "hdrhistogram"
@ -1162,9 +1158,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
[[package]] [[package]]
name = "hermit-abi" name = "hermit-abi"
version = "0.3.5" version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd"
[[package]] [[package]]
name = "hex" name = "hex"
@ -1238,7 +1234,7 @@ dependencies = [
"httpdate", "httpdate",
"itoa", "itoa",
"pin-project-lite", "pin-project-lite",
"socket2 0.5.5", "socket2 0.5.6",
"tokio", "tokio",
"tower-service", "tower-service",
"tracing", "tracing",
@ -1328,9 +1324,9 @@ dependencies = [
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.2.2" version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.14.3", "hashbrown 0.14.3",
@ -1387,9 +1383,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.67" version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee"
dependencies = [ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
@ -1484,9 +1480,9 @@ checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]] [[package]]
name = "metrics" name = "metrics"
version = "0.22.0" version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77b9e10a211c839210fd7f99954bda26e5f8e26ec686ad68da6a32df7c80e782" checksum = "cd71d9db2e4287c3407fa04378b8c2ee570aebe0854431562cdd89ca091854f4"
dependencies = [ dependencies = [
"ahash", "ahash",
"portable-atomic", "portable-atomic",
@ -1494,13 +1490,13 @@ dependencies = [
[[package]] [[package]]
name = "metrics-exporter-prometheus" name = "metrics-exporter-prometheus"
version = "0.13.0" version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83a4c4718a371ddfb7806378f23617876eea8b82e5ff1324516bcd283249d9ea" checksum = "9bf4e7146e30ad172c42c39b3246864bd2d3c6396780711a1baf749cfe423e21"
dependencies = [ dependencies = [
"base64", "base64",
"hyper", "hyper",
"indexmap 1.9.3", "indexmap 2.2.3",
"ipnet", "ipnet",
"metrics", "metrics",
"metrics-util", "metrics-util",
@ -1511,13 +1507,13 @@ dependencies = [
[[package]] [[package]]
name = "metrics-util" name = "metrics-util"
version = "0.16.0" version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2670b8badcc285d486261e2e9f1615b506baff91427b61bd336a472b65bbf5ed" checksum = "ece71ab046dcf45604e573329966ec1db5ff4b81cfa170a924ff4c959ab5451a"
dependencies = [ dependencies = [
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-utils", "crossbeam-utils",
"hashbrown 0.13.1", "hashbrown 0.14.3",
"metrics", "metrics",
"num_cpus", "num_cpus",
"quanta", "quanta",
@ -1610,9 +1606,9 @@ checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]] [[package]]
name = "num-traits" name = "num-traits"
version = "0.2.17" version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
dependencies = [ dependencies = [
"autocfg", "autocfg",
] ]
@ -1650,7 +1646,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"indexmap 2.2.2", "indexmap 2.2.3",
"js-sys", "js-sys",
"once_cell", "once_cell",
"pin-project-lite", "pin-project-lite",
@ -1844,6 +1840,7 @@ dependencies = [
"async-trait", "async-trait",
"barrel", "barrel",
"base64", "base64",
"blurhash-update",
"clap", "clap",
"color-eyre", "color-eyre",
"config", "config",
@ -1870,7 +1867,7 @@ dependencies = [
"reqwest-tracing", "reqwest-tracing",
"rustls 0.22.2", "rustls 0.22.2",
"rustls-channel-resolver", "rustls-channel-resolver",
"rustls-pemfile 2.0.0", "rustls-pemfile 2.1.0",
"rusty-s3", "rusty-s3",
"serde", "serde",
"serde-tuple-vec-map", "serde-tuple-vec-map",
@ -1917,7 +1914,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -2049,7 +2046,7 @@ dependencies = [
"itertools 0.11.0", "itertools 0.11.0",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -2194,7 +2191,7 @@ dependencies = [
"quote", "quote",
"refinery-core", "refinery-core",
"regex", "regex",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -2324,16 +2321,17 @@ checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0"
[[package]] [[package]]
name = "ring" name = "ring"
version = "0.17.7" version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
dependencies = [ dependencies = [
"cc", "cc",
"cfg-if",
"getrandom", "getrandom",
"libc", "libc",
"spin", "spin",
"untrusted", "untrusted",
"windows-sys 0.48.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -2419,9 +2417,9 @@ dependencies = [
[[package]] [[package]]
name = "rustls-pemfile" name = "rustls-pemfile"
version = "2.0.0" version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" checksum = "3c333bb734fcdedcea57de1602543590f545f127dc8b533324318fd492c5c70b"
dependencies = [ dependencies = [
"base64", "base64",
"rustls-pki-types", "rustls-pki-types",
@ -2429,9 +2427,9 @@ dependencies = [
[[package]] [[package]]
name = "rustls-pki-types" name = "rustls-pki-types"
version = "1.2.0" version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" checksum = "048a63e5b3ac996d78d402940b5fa47973d2d080c6c6fffa1d0f19c4445310b7"
[[package]] [[package]]
name = "rustls-webpki" name = "rustls-webpki"
@ -2481,9 +2479,9 @@ dependencies = [
[[package]] [[package]]
name = "ryu" name = "ryu"
version = "1.0.16" version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
[[package]] [[package]]
name = "same-file" name = "same-file"
@ -2528,15 +2526,15 @@ dependencies = [
[[package]] [[package]]
name = "semver" name = "semver"
version = "1.0.21" version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.196" version = "1.0.197"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
dependencies = [ dependencies = [
"serde_derive", "serde_derive",
] ]
@ -2552,20 +2550,20 @@ dependencies = [
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.196" version = "1.0.197"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
name = "serde_json" name = "serde_json"
version = "1.0.113" version = "1.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
dependencies = [ dependencies = [
"itoa", "itoa",
"ryu", "ryu",
@ -2713,12 +2711,12 @@ dependencies = [
[[package]] [[package]]
name = "socket2" name = "socket2"
version = "0.5.5" version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.48.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -2769,9 +2767,9 @@ dependencies = [
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.10.0" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01"
[[package]] [[package]]
name = "subtle" name = "subtle"
@ -2792,9 +2790,9 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.48" version = "2.0.50"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2839,29 +2837,29 @@ dependencies = [
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.56" version = "1.0.57"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.56" version = "1.0.57"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
name = "thread_local" name = "thread_local"
version = "1.1.7" version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"once_cell", "once_cell",
@ -2927,7 +2925,7 @@ dependencies = [
"parking_lot 0.12.1", "parking_lot 0.12.1",
"pin-project-lite", "pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"socket2 0.5.5", "socket2 0.5.6",
"tokio-macros", "tokio-macros",
"tracing", "tracing",
"windows-sys 0.48.0", "windows-sys 0.48.0",
@ -2951,7 +2949,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -2974,7 +2972,7 @@ dependencies = [
"postgres-protocol", "postgres-protocol",
"postgres-types", "postgres-types",
"rand", "rand",
"socket2 0.5.5", "socket2 0.5.6",
"tokio", "tokio",
"tokio-util", "tokio-util",
"whoami", "whoami",
@ -3058,9 +3056,9 @@ dependencies = [
[[package]] [[package]]
name = "toml" name = "toml"
version = "0.8.9" version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6a4b9e8023eb94392d3dca65d717c53abc5dad49c07cb65bb8fcd87115fa325" checksum = "9a9aad4a3066010876e8dcf5a8a06e70a558751117a145c6ce2b82c2e2054290"
dependencies = [ dependencies = [
"serde", "serde",
"serde_spanned", "serde_spanned",
@ -3079,11 +3077,11 @@ dependencies = [
[[package]] [[package]]
name = "toml_edit" name = "toml_edit"
version = "0.21.1" version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6"
dependencies = [ dependencies = [
"indexmap 2.2.2", "indexmap 2.2.3",
"serde", "serde",
"serde_spanned", "serde_spanned",
"toml_datetime", "toml_datetime",
@ -3212,7 +3210,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -3330,9 +3328,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]] [[package]]
name = "unicode-normalization" name = "unicode-normalization"
version = "0.1.22" version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
dependencies = [ dependencies = [
"tinyvec", "tinyvec",
] ]
@ -3417,9 +3415,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.90" version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"wasm-bindgen-macro", "wasm-bindgen-macro",
@ -3427,24 +3425,24 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-backend" name = "wasm-bindgen-backend"
version = "0.2.90" version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b"
dependencies = [ dependencies = [
"bumpalo", "bumpalo",
"log", "log",
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
[[package]] [[package]]
name = "wasm-bindgen-futures" name = "wasm-bindgen-futures"
version = "0.4.40" version = "0.4.41"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys", "js-sys",
@ -3454,9 +3452,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.90" version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed"
dependencies = [ dependencies = [
"quote", "quote",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
@ -3464,22 +3462,22 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro-support" name = "wasm-bindgen-macro-support"
version = "0.2.90" version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
[[package]] [[package]]
name = "wasm-bindgen-shared" name = "wasm-bindgen-shared"
version = "0.2.90" version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838"
[[package]] [[package]]
name = "wasm-streams" name = "wasm-streams"
@ -3496,9 +3494,9 @@ dependencies = [
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.67" version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
@ -3576,7 +3574,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [ dependencies = [
"windows-targets 0.52.0", "windows-targets 0.52.3",
] ]
[[package]] [[package]]
@ -3594,7 +3592,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [ dependencies = [
"windows-targets 0.52.0", "windows-targets 0.52.3",
] ]
[[package]] [[package]]
@ -3614,17 +3612,17 @@ dependencies = [
[[package]] [[package]]
name = "windows-targets" name = "windows-targets"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f"
dependencies = [ dependencies = [
"windows_aarch64_gnullvm 0.52.0", "windows_aarch64_gnullvm 0.52.3",
"windows_aarch64_msvc 0.52.0", "windows_aarch64_msvc 0.52.3",
"windows_i686_gnu 0.52.0", "windows_i686_gnu 0.52.3",
"windows_i686_msvc 0.52.0", "windows_i686_msvc 0.52.3",
"windows_x86_64_gnu 0.52.0", "windows_x86_64_gnu 0.52.3",
"windows_x86_64_gnullvm 0.52.0", "windows_x86_64_gnullvm 0.52.3",
"windows_x86_64_msvc 0.52.0", "windows_x86_64_msvc 0.52.3",
] ]
[[package]] [[package]]
@ -3635,9 +3633,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]] [[package]]
name = "windows_aarch64_gnullvm" name = "windows_aarch64_gnullvm"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
@ -3647,9 +3645,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
@ -3659,9 +3657,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
@ -3671,9 +3669,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
@ -3683,9 +3681,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
@ -3695,9 +3693,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
@ -3707,15 +3705,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
version = "0.52.0" version = "0.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.5.37" version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7cad8365489051ae9f054164e459304af2e7e9bb407c958076c8bf4aef52da5" checksum = "7a4191c47f15cc3ec71fcb4913cb83d58def65dd3787610213c649283b5ce178"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@ -3775,7 +3773,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]
[[package]] [[package]]
@ -3795,5 +3793,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.48", "syn 2.0.50",
] ]

1
Cargo.toml
View file

@ -22,6 +22,7 @@ actix-web = { version = "4.0.0", default-features = false, features = ["rustls-0
async-trait = "0.1.51" async-trait = "0.1.51"
barrel = { version = "0.7.0", features = ["pg"] } barrel = { version = "0.7.0", features = ["pg"] }
base64 = "0.21.0" base64 = "0.21.0"
blurhash-update = { git = "https://git.asonix.dog/asonix/blurhash-update", version = "0.1.0" }
clap = { version = "4.0.2", features = ["derive"] } clap = { version = "4.0.2", features = ["derive"] }
color-eyre = "0.6" color-eyre = "0.6"
config = { version = "0.14.0", default-features = false, features = ["json", "ron", "toml", "yaml"] } config = { version = "0.14.0", default-features = false, features = ["json", "ron", "toml", "yaml"] }

105
src/blurhash.rs Normal file
View file

@ -0,0 +1,105 @@
use std::ffi::{OsStr, OsString};
use tokio::io::AsyncReadExt;
use crate::{
details::Details,
error::{Error, UploadError},
formats::ProcessableFormat,
magick::{MagickError, MAGICK_CONFIGURE_PATH, MAGICK_TEMPORARY_PATH},
process::Process,
repo::Alias,
state::State,
store::Store,
};
pub(crate) async fn generate<S>(
state: &State<S>,
alias: &Alias,
original_details: &Details,
) -> Result<String, Error>
where
S: Store + 'static,
{
let hash = state
.repo
.hash(alias)
.await?
.ok_or(UploadError::MissingIdentifier)?;
let identifier = if original_details.is_video() {
crate::generate::ensure_motion_identifier(state, hash, original_details).await?
} else {
state
.repo
.identifier(hash)
.await?
.ok_or(UploadError::MissingIdentifier)?
};
let input_details = crate::ensure_details_identifier(state, &identifier).await?;
let stream = state.store.to_stream(&identifier, None, None).await?;
let blurhash = read_rgba_command(
state,
input_details
.internal_format()
.processable_format()
.expect("not a video"),
)
.await?
.drive_with_stream(stream)
.with_stdout(|mut stdout| async move {
let mut encoder = blurhash_update::Encoder::auto(blurhash_update::ImageBounds {
width: input_details.width() as _,
height: input_details.height() as _,
});
let mut buf = [0u8; 1024 * 8];
loop {
let n = stdout.read(&mut buf).await?;
if n == 0 {
break;
}
encoder.update(&buf[..n]);
}
Ok(encoder.finalize()) as std::io::Result<String>
})
.await??;
Ok(blurhash)
}
async fn read_rgba_command<S>(
state: &State<S>,
input_format: ProcessableFormat,
) -> Result<Process, MagickError> {
let temporary_path = state
.tmp_dir
.tmp_folder()
.await
.map_err(MagickError::CreateTemporaryDirectory)?;
let mut input_arg = OsString::from(input_format.magick_format());
input_arg.push(":-");
if input_format.coalesce() {
input_arg.push("[0]");
}
let args: [&OsStr; 3] = ["convert".as_ref(), &input_arg, "RGBA:-".as_ref()];
let envs = [
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
];
let process = Process::run("magick", &args, &envs, state.config.media.process_timeout)?
.add_extras(temporary_path);
Ok(process)
}

12
src/details.rs
View file

@ -100,6 +100,14 @@ impl Details {
)) ))
} }
pub(crate) fn width(&self) -> u16 {
self.inner.width
}
pub(crate) fn height(&self) -> u16 {
self.inner.height
}
pub(crate) fn internal_format(&self) -> InternalFormat { pub(crate) fn internal_format(&self) -> InternalFormat {
self.inner.format self.inner.format
} }
@ -112,6 +120,10 @@ impl Details {
self.inner.created_at.into() self.inner.created_at.into()
} }
pub(crate) fn is_video(&self) -> bool {
matches!(self.inner.format, InternalFormat::Video(_))
}
pub(crate) fn video_format(&self) -> Option<InternalVideoFormat> { pub(crate) fn video_format(&self) -> Option<InternalVideoFormat> {
match self.inner.format { match self.inner.format {
InternalFormat::Video(format) => Some(format), InternalFormat::Video(format) => Some(format),

4
src/discover/exiftool.rs
View file

@ -1,5 +1,3 @@
use crate::{ use crate::{
bytes_stream::BytesStream, bytes_stream::BytesStream,
exiftool::ExifError, exiftool::ExifError,
@ -43,7 +41,7 @@ pub(super) async fn check_reorient(
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
async fn needs_reorienting(input: BytesStream, timeout: u64) -> Result<bool, ExifError> { async fn needs_reorienting(input: BytesStream, timeout: u64) -> Result<bool, ExifError> {
let buf = Process::run("exiftool", &["-n", "-Orientation", "-"], &[], timeout)? let buf = Process::run("exiftool", &["-n", "-Orientation", "-"], &[], timeout)?
.bytes_stream_read(input) .drive_with_async_read(input.into_reader())
.into_string() .into_string()
.await?; .await?;

63
src/discover/ffmpeg.rs
View file

@ -14,7 +14,6 @@ use crate::{
state::State, state::State,
}; };
use super::Discovery; use super::Discovery;
const MP4: &str = "mp4"; const MP4: &str = "mp4";
@ -158,24 +157,6 @@ struct Flags {
alpha: usize, alpha: usize,
} }
#[tracing::instrument(skip_all)]
pub(super) async fn discover_bytes_stream<S>(
state: &State<S>,
bytes: BytesStream,
) -> Result<Option<Discovery>, FfMpegError> {
discover_file(state, move |mut file| {
let bytes = bytes.clone();
async move {
file.write_from_stream(bytes.into_io_stream())
.await
.map_err(FfMpegError::Write)?;
Ok(file)
}
})
.await
}
async fn allows_alpha(pixel_format: &str, timeout: u64) -> Result<bool, FfMpegError> { async fn allows_alpha(pixel_format: &str, timeout: u64) -> Result<bool, FfMpegError> {
static ALPHA_PIXEL_FORMATS: OnceLock<HashSet<String>> = OnceLock::new(); static ALPHA_PIXEL_FORMATS: OnceLock<HashSet<String>> = OnceLock::new();
@ -191,45 +172,31 @@ async fn allows_alpha(pixel_format: &str, timeout: u64) -> Result<bool, FfMpegEr
} }
#[tracing::instrument(level = "debug", skip_all)] #[tracing::instrument(level = "debug", skip_all)]
async fn discover_file<S, F, Fut>(state: &State<S>, f: F) -> Result<Option<Discovery>, FfMpegError> pub(super) async fn discover_bytes_stream<S>(
where state: &State<S>,
F: FnOnce(crate::file::File) -> Fut, bytes: BytesStream,
Fut: std::future::Future<Output = Result<crate::file::File, FfMpegError>>, ) -> Result<Option<Discovery>, FfMpegError> {
{
let input_file = state.tmp_dir.tmp_file(None);
crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(FfMpegError::CreateDir)?;
let tmp_one = crate::file::File::create(&input_file)
.await
.map_err(FfMpegError::CreateFile)?;
let tmp_one = (f)(tmp_one).await?;
tmp_one.close().await.map_err(FfMpegError::CloseFile)?;
let res = Process::run( let res = Process::run(
"ffprobe", "ffprobe",
&[ &[
"-v".as_ref(), "-v",
"quiet".as_ref(), "quiet",
"-count_frames".as_ref(), "-count_frames",
"-show_entries".as_ref(), "-show_entries",
"stream=width,height,nb_read_frames,codec_name,pix_fmt:format=format_name".as_ref(), "stream=width,height,nb_read_frames,codec_name,pix_fmt:format=format_name",
"-of".as_ref(), "-of",
"default=noprint_wrappers=1:nokey=1".as_ref(), "default=noprint_wrappers=1:nokey=1",
"-print_format".as_ref(), "-print_format",
"json".as_ref(), "json",
input_file.as_os_str(), "-",
], ],
&[], &[],
state.config.media.process_timeout, state.config.media.process_timeout,
)? )?
.read() .drive_with_async_read(bytes.into_reader())
.into_vec() .into_vec()
.await; .await;
input_file.cleanup().await.map_err(FfMpegError::Cleanup)?;
let output = res?; let output = res?;
let output: FfMpegDiscovery = serde_json::from_slice(&output).map_err(FfMpegError::Json)?; let output: FfMpegDiscovery = serde_json::from_slice(&output).map_err(FfMpegError::Json)?;

37
src/discover/magick.rs
View file

@ -1,8 +1,6 @@
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
use crate::{ use crate::{
bytes_stream::BytesStream, bytes_stream::BytesStream,
discover::DiscoverError, discover::DiscoverError,
@ -50,39 +48,17 @@ pub(super) async fn confirm_bytes_stream<S>(
} }
} }
discover_file(state, move |mut file| async move { discover(state, bytes).await
file.write_from_stream(bytes.into_io_stream())
.await
.map_err(MagickError::Write)?;
Ok(file)
})
.await
} }
#[tracing::instrument(level = "debug", skip_all)] #[tracing::instrument(level = "debug", skip_all)]
async fn discover_file<S, F, Fut>(state: &State<S>, f: F) -> Result<Discovery, MagickError> async fn discover<S>(state: &State<S>, stream: BytesStream) -> Result<Discovery, MagickError> {
where
F: FnOnce(crate::file::File) -> Fut,
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
{
let temporary_path = state let temporary_path = state
.tmp_dir .tmp_dir
.tmp_folder() .tmp_folder()
.await .await
.map_err(MagickError::CreateTemporaryDirectory)?; .map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = state.tmp_dir.tmp_file(None);
crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(MagickError::CreateDir)?;
let tmp_one = crate::file::File::create(&input_file)
.await
.map_err(MagickError::CreateFile)?;
let tmp_one = (f)(tmp_one).await?;
tmp_one.close().await.map_err(MagickError::CloseFile)?;
let envs = [ let envs = [
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()), (MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()), (MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
@ -91,19 +67,16 @@ where
let res = Process::run( let res = Process::run(
"magick", "magick",
&[ &[
"convert".as_ref(), "convert", // "-ping".as_ref(), // re-enable -ping after imagemagick fix
// "-ping".as_ref(), // re-enable -ping after imagemagick fix "-", "JSON:",
input_file.as_os_str(),
"JSON:".as_ref(),
], ],
&envs, &envs,
state.config.media.process_timeout, state.config.media.process_timeout,
)? )?
.read() .drive_with_async_read(stream.into_reader())
.into_string() .into_string()
.await; .await;
input_file.cleanup().await.map_err(MagickError::Cleanup)?;
temporary_path temporary_path
.cleanup() .cleanup()
.await .await

27
src/exiftool.rs
View file

@ -1,9 +1,4 @@
use crate::{ use crate::{error_code::ErrorCode, process::ProcessError};
bytes_stream::BytesStream,
error_code::ErrorCode,
process::{Process, ProcessError, ProcessRead},
};
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub(crate) enum ExifError { pub(crate) enum ExifError {
@ -38,23 +33,3 @@ impl ExifError {
} }
} }
} }
#[tracing::instrument(level = "trace", skip(input))]
pub(crate) async fn needs_reorienting(timeout: u64, input: BytesStream) -> Result<bool, ExifError> {
let buf = Process::run("exiftool", &["-n", "-Orientation", "-"], &[], timeout)?
.bytes_stream_read(input)
.into_string()
.await?;
Ok(!buf.is_empty())
}
#[tracing::instrument(level = "trace", skip(input))]
pub(crate) fn clear_metadata_bytes_read(
timeout: u64,
input: BytesStream,
) -> Result<ProcessRead, ExifError> {
let process = Process::run("exiftool", &["-all=", "-", "-out", "-"], &[], timeout)?;
Ok(process.bytes_stream_read(input))
}

66
src/generate.rs
View file

@ -117,15 +117,10 @@ async fn process<S: Store + 'static>(
let stream = state.store.to_stream(&identifier, None, None).await?; let stream = state.store.to_stream(&identifier, None, None).await?;
let bytes = crate::magick::process_image_stream_read( let bytes =
state, crate::magick::process_image_command(state, thumbnail_args, input_format, format, quality)
stream,
thumbnail_args,
input_format,
format,
quality,
)
.await? .await?
.drive_with_stream(stream)
.into_bytes_stream() .into_bytes_stream()
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
"Reading processed image to BytesStream" "Reading processed image to BytesStream"
@ -160,27 +155,14 @@ async fn process<S: Store + 'static>(
Ok((details, identifier)) as Result<(Details, Arc<str>), Error> Ok((details, identifier)) as Result<(Details, Arc<str>), Error>
} }
#[tracing::instrument(skip_all)] pub(crate) async fn ensure_motion_identifier<S>(
async fn input_identifier<S>(
state: &State<S>, state: &State<S>,
output_format: InputProcessableFormat,
hash: Hash, hash: Hash,
original_details: &Details, original_details: &Details,
) -> Result<Arc<str>, Error> ) -> Result<Arc<str>, Error>
where where
S: Store + 'static, S: Store + 'static,
{ {
let should_thumbnail =
if let Some(input_format) = original_details.internal_format().processable_format() {
let output_format = input_format.process_to(output_format);
input_format.should_thumbnail(output_format)
} else {
// video case
true
};
if should_thumbnail {
if let Some(identifier) = state.repo.motion_identifier(hash.clone()).await? { if let Some(identifier) = state.repo.motion_identifier(hash.clone()).await? {
return Ok(identifier); return Ok(identifier);
}; };
@ -191,17 +173,19 @@ where
.await? .await?
.ok_or(UploadError::MissingIdentifier)?; .ok_or(UploadError::MissingIdentifier)?;
let (reader, media_type) = if let Some(processable_format) = let (reader, media_type) =
original_details.internal_format().processable_format() if let Some(processable_format) = original_details.internal_format().processable_format() {
{
let thumbnail_format = state.config.media.image.format.unwrap_or(ImageFormat::Webp); let thumbnail_format = state.config.media.image.format.unwrap_or(ImageFormat::Webp);
let stream = state.store.to_stream(&identifier, None, None).await?; let stream = state.store.to_stream(&identifier, None, None).await?;
let reader = let process =
magick::thumbnail(state, stream, processable_format, thumbnail_format).await?; magick::thumbnail_command(state, processable_format, thumbnail_format).await?;
(reader, thumbnail_format.media_type()) (
process.drive_with_stream(stream),
thumbnail_format.media_type(),
)
} else { } else {
let thumbnail_format = match state.config.media.image.format { let thumbnail_format = match state.config.media.image.format {
Some(ImageFormat::Webp | ImageFormat::Avif | ImageFormat::Jxl) => { Some(ImageFormat::Webp | ImageFormat::Avif | ImageFormat::Jxl) => {
@ -233,7 +217,31 @@ where
.relate_motion_identifier(hash, &motion_identifier) .relate_motion_identifier(hash, &motion_identifier)
.await?; .await?;
return Ok(motion_identifier); Ok(motion_identifier)
}
#[tracing::instrument(skip_all)]
async fn input_identifier<S>(
state: &State<S>,
output_format: InputProcessableFormat,
hash: Hash,
original_details: &Details,
) -> Result<Arc<str>, Error>
where
S: Store + 'static,
{
let should_thumbnail =
if let Some(input_format) = original_details.internal_format().processable_format() {
let output_format = input_format.process_to(output_format);
input_format.should_thumbnail(output_format)
} else {
// video case
true
};
if should_thumbnail {
return ensure_motion_identifier(state, hash.clone(), original_details).await;
} }
state state

60
src/generate/magick.rs
View file

@ -1,25 +1,17 @@
use std::ffi::OsStr; use std::ffi::OsStr;
use actix_web::web::Bytes;
use crate::{ use crate::{
formats::{ImageFormat, ProcessableFormat}, formats::{ImageFormat, ProcessableFormat},
magick::{MagickError, MAGICK_CONFIGURE_PATH, MAGICK_TEMPORARY_PATH}, magick::{MagickError, MAGICK_CONFIGURE_PATH, MAGICK_TEMPORARY_PATH},
process::{Process, ProcessRead}, process::Process,
state::State, state::State,
stream::LocalBoxStream,
}; };
async fn thumbnail_animation<S, F, Fut>( pub(super) async fn thumbnail_command<S>(
state: &State<S>, state: &State<S>,
input_format: ProcessableFormat, input_format: ProcessableFormat,
thumbnail_format: ImageFormat, thumbnail_format: ImageFormat,
write_file: F, ) -> Result<Process, MagickError> {
) -> Result<ProcessRead, MagickError>
where
F: FnOnce(crate::file::File) -> Fut,
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
{
let format = ProcessableFormat::Image(thumbnail_format); let format = ProcessableFormat::Image(thumbnail_format);
let quality = state.config.media.image.quality_for(thumbnail_format); let quality = state.config.media.image.quality_for(thumbnail_format);
@ -29,22 +21,7 @@ where
.await .await
.map_err(MagickError::CreateTemporaryDirectory)?; .map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = state.tmp_dir.tmp_file(None); let input_arg = format!("{}:-", input_format.magick_format());
crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(MagickError::CreateDir)?;
let tmp_one = crate::file::File::create(&input_file)
.await
.map_err(MagickError::CreateFile)?;
let tmp_one = (write_file)(tmp_one).await?;
tmp_one.close().await.map_err(MagickError::CloseFile)?;
let input_arg = [
input_format.magick_format().as_ref(),
input_file.as_os_str(),
]
.join(":".as_ref());
let output_arg = format!("{}:-", format.magick_format()); let output_arg = format!("{}:-", format.magick_format());
let quality = quality.map(|q| q.to_string()); let quality = quality.map(|q| q.to_string());
@ -52,7 +29,7 @@ where
let mut args: Vec<&OsStr> = Vec::with_capacity(len); let mut args: Vec<&OsStr> = Vec::with_capacity(len);
args.push("convert".as_ref()); args.push("convert".as_ref());
args.push(&input_arg); args.push(input_arg.as_ref());
if format.coalesce() { if format.coalesce() {
args.push("-coalesce".as_ref()); args.push("-coalesce".as_ref());
} }
@ -66,31 +43,8 @@ where
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()), (MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
]; ];
let reader = Process::run("magick", &args, &envs, state.config.media.process_timeout)? let process = Process::run("magick", &args, &envs, state.config.media.process_timeout)?
.read()
.add_extras(input_file)
.add_extras(temporary_path); .add_extras(temporary_path);
Ok(reader) Ok(process)
}
pub(super) async fn thumbnail<S>(
state: &State<S>,
stream: LocalBoxStream<'static, std::io::Result<Bytes>>,
input_format: ProcessableFormat,
thumbnail_format: ImageFormat,
) -> Result<ProcessRead, MagickError> {
thumbnail_animation(
state,
input_format,
thumbnail_format,
|mut tmp_file| async move {
tmp_file
.write_from_stream(stream)
.await
.map_err(MagickError::Write)?;
Ok(tmp_file)
},
)
.await
} }

14
src/ingest.rs
View file

@ -69,15 +69,11 @@ where
} }
}; };
crate::magick::process_image_process_read( let process =
state, crate::magick::process_image_command(state, magick_args, format, format, quality)
process_read, .await?;
magick_args,
format, process_read.pipe(process)
format,
quality,
)
.await?
} else { } else {
process_read process_read
} }

40
src/lib.rs
View file

@ -1,4 +1,5 @@
mod backgrounded; mod backgrounded;
mod blurhash;
mod bytes_stream; mod bytes_stream;
mod concurrent_processor; mod concurrent_processor;
mod config; mod config;
@ -1277,6 +1278,44 @@ fn srv_head(
builder builder
} }
async fn blurhash<S: Store + 'static>(
web::Query(alias_query): web::Query<AliasQuery>,
state: web::Data<State<S>>,
) -> Result<HttpResponse, Error> {
let alias = match alias_query {
AliasQuery::Alias { alias } => Serde::into_inner(alias),
AliasQuery::Proxy { proxy } => {
let alias = if let Some(alias) = state.repo.related(proxy.clone()).await? {
alias
} else if !state.config.server.read_only {
let stream = download_stream(proxy.as_str(), &state).await?;
let (alias, _, _) = ingest_inline(stream, &state).await?;
state.repo.relate_url(proxy, alias.clone()).await?;
alias
} else {
return Err(UploadError::ReadOnly.into());
};
if !state.config.server.read_only {
state.repo.accessed_alias(alias.clone()).await?;
}
alias
}
};
let details = ensure_details(&state, &alias).await?;
let blurhash = blurhash::generate(&state, &alias, &details).await?;
Ok(HttpResponse::Ok().json(serde_json::json!({
"msg": "ok",
"blurhash": blurhash,
})))
}
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
struct PruneResponse { struct PruneResponse {
complete: bool, complete: bool,
@ -1561,6 +1600,7 @@ fn configure_endpoints<S: Store + 'static, F: Fn(&mut web::ServiceConfig)>(
.route(web::head().to(serve_head::<S>)), .route(web::head().to(serve_head::<S>)),
), ),
) )
.service(web::resource("/blurhash").route(web::get().to(blurhash::<S>)))
.service( .service(
web::resource("/process.{ext}") web::resource("/process.{ext}")
.route(web::get().to(process::<S>)) .route(web::get().to(process::<S>))

114
src/magick.rs
View file

@ -1,14 +1,11 @@
use std::{ffi::OsStr, ops::Deref, path::Path, sync::Arc}; use std::{ffi::OsStr, ops::Deref, path::Path, sync::Arc};
use actix_web::web::Bytes;
use crate::{ use crate::{
config::Media, config::Media,
error_code::ErrorCode, error_code::ErrorCode,
formats::ProcessableFormat, formats::ProcessableFormat,
process::{Process, ProcessError, ProcessRead}, process::{Process, ProcessError},
state::State, state::State,
stream::LocalBoxStream,
tmp_file::{TmpDir, TmpFolder}, tmp_file::{TmpDir, TmpFolder},
}; };
@ -23,21 +20,9 @@ pub(crate) enum MagickError {
#[error("Invalid output format: {0}")] #[error("Invalid output format: {0}")]
Json(String, #[source] serde_json::Error), Json(String, #[source] serde_json::Error),
#[error("Error writing bytes")]
Write(#[source] std::io::Error),
#[error("Error creating file")]
CreateFile(#[source] std::io::Error),
#[error("Error creating directory")]
CreateDir(#[source] crate::store::file_store::FileError),
#[error("Error creating temporary directory")] #[error("Error creating temporary directory")]
CreateTemporaryDirectory(#[source] std::io::Error), CreateTemporaryDirectory(#[source] std::io::Error),
#[error("Error closing file")]
CloseFile(#[source] std::io::Error),
#[error("Error in metadata discovery")] #[error("Error in metadata discovery")]
Discover(#[source] crate::discover::DiscoverError), Discover(#[source] crate::discover::DiscoverError),
@ -66,11 +51,7 @@ impl MagickError {
Self::CommandFailed(_) => ErrorCode::COMMAND_FAILURE, Self::CommandFailed(_) => ErrorCode::COMMAND_FAILURE,
Self::Process(e) => e.error_code(), Self::Process(e) => e.error_code(),
Self::Json(_, _) Self::Json(_, _)
| Self::Write(_)
| Self::CreateFile(_)
| Self::CreateDir(_)
| Self::CreateTemporaryDirectory(_) | Self::CreateTemporaryDirectory(_)
| Self::CloseFile(_)
| Self::Discover(_) | Self::Discover(_)
| Self::Cleanup(_) | Self::Cleanup(_)
| Self::Empty => ErrorCode::COMMAND_ERROR, | Self::Empty => ErrorCode::COMMAND_ERROR,
@ -86,40 +67,20 @@ impl MagickError {
} }
} }
async fn process_image<S, F, Fut>( pub(crate) async fn process_image_command<S>(
state: &State<S>, state: &State<S>,
process_args: Vec<String>, process_args: Vec<String>,
input_format: ProcessableFormat, input_format: ProcessableFormat,
format: ProcessableFormat, format: ProcessableFormat,
quality: Option<u8>, quality: Option<u8>,
write_file: F, ) -> Result<Process, MagickError> {
) -> Result<ProcessRead, MagickError>
where
F: FnOnce(crate::file::File) -> Fut,
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
{
let temporary_path = state let temporary_path = state
.tmp_dir .tmp_dir
.tmp_folder() .tmp_folder()
.await .await
.map_err(MagickError::CreateTemporaryDirectory)?; .map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = state.tmp_dir.tmp_file(None); let input_arg = format!("{}:-", input_format.magick_format());
crate::store::file_store::safe_create_parent(&input_file)
.await
.map_err(MagickError::CreateDir)?;
let tmp_one = crate::file::File::create(&input_file)
.await
.map_err(MagickError::CreateFile)?;
let tmp_one = (write_file)(tmp_one).await?;
tmp_one.close().await.map_err(MagickError::CloseFile)?;
let input_arg = [
input_format.magick_format().as_ref(),
input_file.as_os_str(),
]
.join(":".as_ref());
let output_arg = format!("{}:-", format.magick_format()); let output_arg = format!("{}:-", format.magick_format());
let quality = quality.map(|q| q.to_string()); let quality = quality.map(|q| q.to_string());
@ -130,7 +91,7 @@ where
let mut args: Vec<&OsStr> = Vec::with_capacity(len); let mut args: Vec<&OsStr> = Vec::with_capacity(len);
args.push("convert".as_ref()); args.push("convert".as_ref());
args.push(&input_arg); args.push(input_arg.as_ref());
if input_format.coalesce() { if input_format.coalesce() {
args.push("-coalesce".as_ref()); args.push("-coalesce".as_ref());
} }
@ -145,67 +106,10 @@ where
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()), (MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
]; ];
let reader = Process::run("magick", &args, &envs, state.config.media.process_timeout)? let process = Process::run("magick", &args, &envs, state.config.media.process_timeout)?
.read()
.add_extras(input_file)
.add_extras(temporary_path); .add_extras(temporary_path);
Ok(reader) Ok(process)
}
pub(crate) async fn process_image_stream_read<S>(
state: &State<S>,
stream: LocalBoxStream<'static, std::io::Result<Bytes>>,
args: Vec<String>,
input_format: ProcessableFormat,
format: ProcessableFormat,
quality: Option<u8>,
) -> Result<ProcessRead, MagickError> {
process_image(
state,
args,
input_format,
format,
quality,
|mut tmp_file| async move {
tmp_file
.write_from_stream(stream)
.await
.map_err(MagickError::Write)?;
Ok(tmp_file)
},
)
.await
}
pub(crate) async fn process_image_process_read<S>(
state: &State<S>,
process_read: ProcessRead,
args: Vec<String>,
input_format: ProcessableFormat,
format: ProcessableFormat,
quality: Option<u8>,
) -> Result<ProcessRead, MagickError> {
process_image(
state,
args,
input_format,
format,
quality,
|mut tmp_file| async move {
process_read
.with_stdout(|stdout| async {
tmp_file
.write_from_async_read(stdout)
.await
.map_err(MagickError::Write)
})
.await??;
Ok(tmp_file)
},
)
.await
} }
pub(crate) type ArcPolicyDir = Arc<PolicyDir>; pub(crate) type ArcPolicyDir = Arc<PolicyDir>;
@ -280,12 +184,12 @@ fn generate_policy(media: &Media) -> String {
<policy domain="cache" name="synchronize" value="true"/> <policy domain="cache" name="synchronize" value="true"/>
<policy domain="path" rights="none" pattern="@*" /> <policy domain="path" rights="none" pattern="@*" />
<policy domain="coder" rights="none" pattern="*" /> <policy domain="coder" rights="none" pattern="*" />
<policy domain="coder" rights="read | write" pattern="{{APNG,AVIF,GIF,HEIC,JPEG,JSON,JXL,PNG,WEBP,MP4,WEBM,TMP,PAM}}" /> <policy domain="coder" rights="read | write" pattern="{{APNG,AVIF,GIF,HEIC,JPEG,JSON,JXL,PNG,RGB,RGBA,WEBP,MP4,WEBM,TMP,PAM}}" />
<policy domain="delegate" rights="none" pattern="*" /> <policy domain="delegate" rights="none" pattern="*" />
<policy domain="delegate" rights="execute" pattern="FFMPEG" /> <policy domain="delegate" rights="execute" pattern="FFMPEG" />
<policy domain="filter" rights="none" pattern="*" /> <policy domain="filter" rights="none" pattern="*" />
<policy domain="module" rights="none" pattern="*" /> <policy domain="module" rights="none" pattern="*" />
<policy domain="module" rights="read | write" pattern="{{APNG,AVIF,GIF,HEIC,JPEG,JSON,JXL,PNG,WEBP,TMP,PAM,PNM,VIDEO}}" /> <policy domain="module" rights="read | write" pattern="{{APNG,AVIF,GIF,HEIC,JPEG,JSON,JXL,PNG,RGB,RGBA,WEBP,TMP,PAM,PNM,VIDEO}}" />
<!-- indirect reads not permitted --> <!-- indirect reads not permitted -->
<policy domain="system" name="max-memory-request" value="256MiB"/> <policy domain="system" name="max-memory-request" value="256MiB"/>
<policy domain="system" name="memory-map" value="anonymous"/> <policy domain="system" name="memory-map" value="anonymous"/>

108
src/process.rs
View file

@ -6,11 +6,13 @@ use std::{
time::{Duration, Instant}, time::{Duration, Instant},
}; };
use futures_core::Stream;
use streem::IntoStreamer;
use tokio::{ use tokio::{
io::AsyncReadExt, io::{AsyncRead, AsyncReadExt, AsyncWriteExt},
process::{Child, ChildStdin, Command}, process::{Child, ChildStdin, Command},
}; };
use tokio_util::io::ReaderStream; use tokio_util::{bytes::Bytes, io::ReaderStream};
use tracing::Instrument; use tracing::Instrument;
use uuid::Uuid; use uuid::Uuid;
@ -65,6 +67,7 @@ pub(crate) struct Process {
child: Child, child: Child,
guard: MetricsGuard, guard: MetricsGuard,
timeout: Duration, timeout: Duration,
extras: Box<dyn Extras>,
id: Uuid, id: Uuid,
} }
@ -202,11 +205,19 @@ impl Process {
command, command,
guard, guard,
timeout: Duration::from_secs(timeout), timeout: Duration::from_secs(timeout),
extras: Box::new(()),
id: Uuid::now_v7(), id: Uuid::now_v7(),
}) })
}) })
} }
pub(crate) fn add_extras(self, extra: impl Extras + 'static) -> Self {
Self {
extras: Box::new((self.extras, extra)),
..self
}
}
#[tracing::instrument(skip(self), fields(command = %self.command, id = %self.id))] #[tracing::instrument(skip(self), fields(command = %self.command, id = %self.id))]
pub(crate) async fn wait(self) -> Result<(), ProcessError> { pub(crate) async fn wait(self) -> Result<(), ProcessError> {
let Process { let Process {
@ -214,11 +225,17 @@ impl Process {
mut child, mut child,
guard, guard,
timeout, timeout,
mut extras,
id: _, id: _,
} = self; } = self;
let res = child.wait().with_timeout(timeout).await; let res = child.wait().with_timeout(timeout).await;
extras
.consume()
.await
.map_err(|e| ProcessError::Cleanup(command.clone(), e))?;
match res { match res {
Ok(Ok(status)) if status.success() => { Ok(Ok(status)) if status.success() => {
guard.disarm(); guard.disarm();
@ -234,10 +251,12 @@ impl Process {
} }
} }
pub(crate) fn bytes_stream_read(self, input: BytesStream) -> ProcessRead { pub(crate) fn drive_with_async_read(self, input: impl AsyncRead + 'static) -> ProcessRead {
self.spawn_fn(move |mut stdin| { self.drive(move |mut stdin| {
async move { async move {
match tokio::io::copy(&mut input.into_reader(), &mut stdin).await { let mut input = std::pin::pin!(input);
match tokio::io::copy(&mut input, &mut stdin).await {
Ok(_) => Ok(()), Ok(_) => Ok(()),
// BrokenPipe means we finished reading from Stdout, so we don't need to write // BrokenPipe means we finished reading from Stdout, so we don't need to write
// to stdin. We'll still error out if the command failed so treat this as a // to stdin. We'll still error out if the command failed so treat this as a
@ -249,14 +268,34 @@ impl Process {
}) })
} }
pub(crate) fn drive_with_stream<S>(self, input: S) -> ProcessRead
where
S: Stream<Item = std::io::Result<Bytes>> + 'static,
{
self.drive(move |mut stdin| async move {
let stream = std::pin::pin!(input);
let mut stream = stream.into_streamer();
while let Some(mut bytes) = stream.try_next().await? {
match stdin.write_all_buf(&mut bytes).await {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => break,
Err(e) => return Err(e),
}
}
Ok(())
})
}
pub(crate) fn read(self) -> ProcessRead { pub(crate) fn read(self) -> ProcessRead {
self.spawn_fn(|_| async { Ok(()) }) self.drive(|_| async { Ok(()) })
} }
#[allow(unknown_lints)] #[allow(unknown_lints)]
#[allow(clippy::let_with_type_underscore)] #[allow(clippy::let_with_type_underscore)]
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
fn spawn_fn<F, Fut>(self, f: F) -> ProcessRead fn drive<F, Fut>(self, f: F) -> ProcessRead
where where
F: FnOnce(ChildStdin) -> Fut + 'static, F: FnOnce(ChildStdin) -> Fut + 'static,
Fut: Future<Output = std::io::Result<()>>, Fut: Future<Output = std::io::Result<()>>,
@ -266,6 +305,7 @@ impl Process {
mut child, mut child,
guard, guard,
timeout, timeout,
extras,
id, id,
} = self; } = self;
@ -302,7 +342,7 @@ impl Process {
handle, handle,
command, command,
id, id,
extras: Box::new(()), extras,
} }
} }
} }
@ -358,6 +398,58 @@ impl ProcessRead {
.await? .await?
} }
pub(crate) fn pipe(self, process: Process) -> ProcessRead {
let Process {
command,
mut child,
guard,
timeout,
extras,
id,
} = process;
let mut stdin = child.stdin.take().expect("stdin exists");
let stdout = child.stdout.take().expect("stdout exists");
let command2 = command.clone();
let handle = Box::pin(async move {
self.with_stdout(move |mut stdout| async move {
let child_fut = async {
tokio::io::copy(&mut stdout, &mut stdin).await?;
drop(stdout);
drop(stdin);
child.wait().await
};
match child_fut.with_timeout(timeout).await {
Ok(Ok(status)) if status.success() => {
guard.disarm();
Ok(())
}
Ok(Ok(status)) => Err(ProcessError::Status(command2, status)),
Ok(Err(e)) => Err(ProcessError::Other(command2, e)),
Err(_) => {
child
.kill()
.await
.map_err(|e| ProcessError::Other(command2.clone(), e))?;
Err(ProcessError::Timeout(command2))
}
}
})
.await?
});
ProcessRead {
reader: Box::pin(stdout),
handle,
command,
id,
extras,
}
}
pub(crate) async fn with_stdout<Fut>( pub(crate) async fn with_stdout<Fut>(
self, self,
f: impl FnOnce(BoxRead<'static>) -> Fut, f: impl FnOnce(BoxRead<'static>) -> Fut,

57
src/validate.rs
View file

@ -11,11 +11,10 @@ use crate::{
AnimationFormat, AnimationOutput, ImageInput, ImageOutput, InputFile, InputVideoFormat, AnimationFormat, AnimationOutput, ImageInput, ImageOutput, InputFile, InputVideoFormat,
InternalFormat, InternalFormat,
}, },
process::ProcessRead, process::{Process, ProcessRead},
state::State, state::State,
}; };
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub(crate) enum ValidationError { pub(crate) enum ValidationError {
#[error("Too wide")] #[error("Too wide")]
@ -74,15 +73,23 @@ pub(crate) async fn validate_bytes_stream<S>(
match &input { match &input {
InputFile::Image(input) => { InputFile::Image(input) => {
let (format, process_read) = process_image(state, bytes, *input, width, height).await?; let (format, process) =
process_image_command(state, *input, bytes.len(), width, height).await?;
Ok((format, process_read)) Ok((format, process.drive_with_async_read(bytes.into_reader())))
} }
InputFile::Animation(input) => { InputFile::Animation(input) => {
let (format, process_read) = let (format, process) = process_animation_command(
process_animation(state, bytes, *input, width, height, frames.unwrap_or(1)).await?; state,
*input,
bytes.len(),
width,
height,
frames.unwrap_or(1),
)
.await?;
Ok((format, process_read)) Ok((format, process.drive_with_async_read(bytes.into_reader())))
} }
InputFile::Video(input) => { InputFile::Video(input) => {
let (format, process_read) = let (format, process_read) =
@ -93,14 +100,14 @@ pub(crate) async fn validate_bytes_stream<S>(
} }
} }
#[tracing::instrument(skip(state, bytes), fields(len = bytes.len()))] #[tracing::instrument(skip(state))]
async fn process_image<S>( async fn process_image_command<S>(
state: &State<S>, state: &State<S>,
bytes: BytesStream,
input: ImageInput, input: ImageInput,
length: usize,
width: u16, width: u16,
height: u16, height: u16,
) -> Result<(InternalFormat, ProcessRead), Error> { ) -> Result<(InternalFormat, Process), Error> {
let validations = &state.config.media.image; let validations = &state.config.media.image;
if width > validations.max_width { if width > validations.max_width {
@ -112,7 +119,7 @@ async fn process_image<S>(
if u32::from(width) * u32::from(height) > validations.max_area { if u32::from(width) * u32::from(height) > validations.max_area {
return Err(ValidationError::Area.into()); return Err(ValidationError::Area.into());
} }
if bytes.len() > validations.max_file_size * MEGABYTES { if length > validations.max_file_size * MEGABYTES {
return Err(ValidationError::Filesize.into()); return Err(ValidationError::Filesize.into());
} }
@ -121,15 +128,15 @@ async fn process_image<S>(
needs_transcode, needs_transcode,
} = input.build_output(validations.format); } = input.build_output(validations.format);
let process_read = if needs_transcode { let process = if needs_transcode {
let quality = validations.quality_for(format); let quality = validations.quality_for(format);
magick::convert_image(state, input.format, format, quality, bytes).await? magick::convert_image_command(state, input.format, format, quality).await?
} else { } else {
exiftool::clear_metadata_bytes_read(bytes, state.config.media.process_timeout)? exiftool::clear_metadata_command(state.config.media.process_timeout)?
}; };
Ok((InternalFormat::Image(format), process_read)) Ok((InternalFormat::Image(format), process))
} }
fn validate_animation( fn validate_animation(
@ -158,33 +165,33 @@ fn validate_animation(
Ok(()) Ok(())
} }
#[tracing::instrument(skip(state, bytes), fields(len = bytes.len()))] #[tracing::instrument(skip(state))]
async fn process_animation<S>( async fn process_animation_command<S>(
state: &State<S>, state: &State<S>,
bytes: BytesStream,
input: AnimationFormat, input: AnimationFormat,
length: usize,
width: u16, width: u16,
height: u16, height: u16,
frames: u32, frames: u32,
) -> Result<(InternalFormat, ProcessRead), Error> { ) -> Result<(InternalFormat, Process), Error> {
let validations = &state.config.media.animation; let validations = &state.config.media.animation;
validate_animation(bytes.len(), width, height, frames, validations)?; validate_animation(length, width, height, frames, validations)?;
let AnimationOutput { let AnimationOutput {
format, format,
needs_transcode, needs_transcode,
} = input.build_output(validations.format); } = input.build_output(validations.format);
let process_read = if needs_transcode { let process = if needs_transcode {
let quality = validations.quality_for(format); let quality = validations.quality_for(format);
magick::convert_animation(state, input, format, quality, bytes).await? magick::convert_animation_command(state, input, format, quality).await?
} else { } else {
exiftool::clear_metadata_bytes_read(bytes, state.config.media.process_timeout)? exiftool::clear_metadata_command(state.config.media.process_timeout)?
}; };
Ok((InternalFormat::Animation(format), process_read)) Ok((InternalFormat::Animation(format), process))
} }
fn validate_video( fn validate_video(

23
src/validate/exiftool.rs
View file

@ -1,18 +1,11 @@
use crate::{exiftool::ExifError, process::Process};
use crate::{
bytes_stream::BytesStream,
exiftool::ExifError,
process::{Process, ProcessRead},
};
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
pub(crate) fn clear_metadata_bytes_read( pub(super) fn clear_metadata_command(timeout: u64) -> Result<Process, ExifError> {
input: BytesStream, Ok(Process::run(
timeout: u64, "exiftool",
) -> Result<ProcessRead, ExifError> { &["-all=", "-", "-out", "-"],
Ok( &[],
Process::run("exiftool", &["-all=", "-", "-out", "-"], &[], timeout)? timeout,
.bytes_stream_read(input), )?)
)
} }

14
src/validate/ffmpeg.rs
View file

@ -1,6 +1,5 @@
use std::{ffi::OsStr, sync::Arc}; use std::{ffi::OsStr, sync::Arc};
use uuid::Uuid; use uuid::Uuid;
use crate::{ use crate::{
@ -35,6 +34,7 @@ pub(super) async fn transcode_bytes(
let output_file = tmp_dir.tmp_file(None); let output_file = tmp_dir.tmp_file(None);
let res = async {
let res = transcode_files( let res = transcode_files(
input_file.as_os_str(), input_file.as_os_str(),
input_format, input_format,
@ -55,7 +55,17 @@ pub(super) async fn transcode_bytes(
.read_to_stream(None, None) .read_to_stream(None, None)
.await .await
.map_err(FfMpegError::ReadFile)?; .map_err(FfMpegError::ReadFile)?;
let reader = tokio_util::io::StreamReader::new(stream); Ok(tokio_util::io::StreamReader::new(stream))
}
.await;
let reader = match res {
Ok(reader) => reader,
Err(e) => {
output_file.cleanup().await.map_err(FfMpegError::Cleanup)?;
return Err(e);
}
};
let process_read = ProcessRead::new( let process_read = ProcessRead::new(
Box::pin(reader), Box::pin(reader),

55
src/validate/magick.rs
View file

@ -1,82 +1,60 @@
use std::ffi::OsStr; use std::ffi::OsStr;
use crate::{ use crate::{
bytes_stream::BytesStream,
formats::{AnimationFormat, ImageFormat}, formats::{AnimationFormat, ImageFormat},
magick::{MagickError, MAGICK_CONFIGURE_PATH, MAGICK_TEMPORARY_PATH}, magick::{MagickError, MAGICK_CONFIGURE_PATH, MAGICK_TEMPORARY_PATH},
process::{Process, ProcessRead}, process::Process,
state::State, state::State,
}; };
pub(super) async fn convert_image<S>( pub(super) async fn convert_image_command<S>(
state: &State<S>, state: &State<S>,
input: ImageFormat, input: ImageFormat,
output: ImageFormat, output: ImageFormat,
quality: Option<u8>, quality: Option<u8>,
bytes: BytesStream, ) -> Result<Process, MagickError> {
) -> Result<ProcessRead, MagickError> {
convert( convert(
state, state,
input.magick_format(), input.magick_format(),
output.magick_format(), output.magick_format(),
false, false,
quality, quality,
bytes,
) )
.await .await
} }
pub(super) async fn convert_animation<S>( pub(super) async fn convert_animation_command<S>(
state: &State<S>, state: &State<S>,
input: AnimationFormat, input: AnimationFormat,
output: AnimationFormat, output: AnimationFormat,
quality: Option<u8>, quality: Option<u8>,
bytes: BytesStream, ) -> Result<Process, MagickError> {
) -> Result<ProcessRead, MagickError> {
convert( convert(
state, state,
input.magick_format(), input.magick_format(),
output.magick_format(), output.magick_format(),
true, true,
quality, quality,
bytes,
) )
.await .await
} }
async fn convert<S>( async fn convert<S>(
state: &State<S>, state: &State<S>,
input: &'static str, input_format: &'static str,
output: &'static str, output_format: &'static str,
coalesce: bool, coalesce: bool,
quality: Option<u8>, quality: Option<u8>,
bytes: BytesStream, ) -> Result<Process, MagickError> {
) -> Result<ProcessRead, MagickError> {
let temporary_path = state let temporary_path = state
.tmp_dir .tmp_dir
.tmp_folder() .tmp_folder()
.await .await
.map_err(MagickError::CreateTemporaryDirectory)?; .map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = state.tmp_dir.tmp_file(None); let input_arg = format!("{input_format}:-");
crate::store::file_store::safe_create_parent(&input_file) let output_arg = format!("{output_format}:-");
.await
.map_err(MagickError::CreateDir)?;
let mut tmp_one = crate::file::File::create(&input_file)
.await
.map_err(MagickError::CreateFile)?;
tmp_one
.write_from_stream(bytes.into_io_stream())
.await
.map_err(MagickError::Write)?;
tmp_one.close().await.map_err(MagickError::CloseFile)?;
let input_arg = [input.as_ref(), input_file.as_os_str()].join(":".as_ref());
let output_arg = format!("{output}:-");
let quality = quality.map(|q| q.to_string()); let quality = quality.map(|q| q.to_string());
let mut args: Vec<&OsStr> = vec!["convert".as_ref()]; let mut args: Vec<&OsStr> = vec!["convert".as_ref()];
@ -85,7 +63,11 @@ async fn convert<S>(
args.push("-coalesce".as_ref()); args.push("-coalesce".as_ref());
} }
args.extend(["-strip".as_ref(), "-auto-orient".as_ref(), &input_arg] as [&OsStr; 3]); args.extend([
"-strip".as_ref(),
"-auto-orient".as_ref(),
input_arg.as_ref(),
] as [&OsStr; 3]);
if let Some(quality) = &quality { if let Some(quality) = &quality {
args.extend(["-quality".as_ref(), quality.as_ref()] as [&OsStr; 2]); args.extend(["-quality".as_ref(), quality.as_ref()] as [&OsStr; 2]);
@ -98,9 +80,8 @@ async fn convert<S>(
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()), (MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
]; ];
let reader = Process::run("magick", &args, &envs, state.config.media.process_timeout)?.read(); let process = Process::run("magick", &args, &envs, state.config.media.process_timeout)?
.add_extras(temporary_path);
let clean_reader = reader.add_extras(input_file).add_extras(temporary_path); Ok(process)
Ok(clean_reader)
} }