LemmyNet
/
pict-rs
mirror of https://git.asonix.dog/asonix/pict-rs
2
0
Fork 0
Code Releases Activity

Merge pull request 'asonix/postgres-repo' (#39) from asonix/postgres-repo into main 

Reviewed-on: https://git.asonix.dog/asonix/pict-rs/pulls/39
This commit is contained in:
asonix 2023-09-05 03:30:49 +00:00
parent e4b264be15 bc49f8ca37
commit 509a52ec6b
64 changed files with 4297 additions and 1374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

389
Cargo.lock generated
View File

@ -77,7 +77,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -313,7 +313,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -388,6 +388,12 @@ dependencies = [
"rustc-demangle",
]
[[package]]
name = "barrel"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad9e605929a6964efbec5ac0884bd0fe93f12a3b1eb271f52c251316640c68d9"
[[package]]
name = "base64"
version = "0.13.1"
@ -494,7 +500,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -670,6 +676,28 @@ dependencies = [
"parking_lot_core 0.9.8",
]
[[package]]
name = "deadpool"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e"
dependencies = [
"async-trait",
"deadpool-runtime",
"num_cpus",
"retain_mut",
"tokio",
]
[[package]]
name = "deadpool-runtime"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1"
dependencies = [
"tokio",
]
[[package]]
name = "deranged"
version = "0.3.8"
@ -692,6 +720,69 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "diesel"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98235fdc2f355d330a8244184ab6b4b33c28679c0b4158f63138e51d6cf7e88"
dependencies = [
"bitflags 2.4.0",
"byteorder",
"diesel_derives",
"itoa",
"serde_json",
"time",
"uuid",
]
[[package]]
name = "diesel-async"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acada1517534c92d3f382217b485db8a8638f111b0e3f2a2a8e26165050f77be"
dependencies = [
"async-trait",
"deadpool",
"diesel",
"futures-util",
"scoped-futures",
"tokio",
"tokio-postgres",
]
[[package]]
name = "diesel-derive-enum"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81c5131a2895ef64741dad1d483f358c2a229a3a2d1b256778cdc5e146db64d4"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.31",
]
[[package]]
name = "diesel_derives"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e054665eaf6d97d1e7125512bb2d35d07c73ac86cc6920174cb42d1ab697a554"
dependencies = [
"diesel_table_macro_syntax",
"proc-macro2",
"quote",
"syn 2.0.31",
]
[[package]]
name = "diesel_table_macro_syntax"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
dependencies = [
"syn 2.0.31",
]
[[package]]
name = "digest"
version = "0.10.7"
@ -740,6 +831,12 @@ dependencies = [
"once_cell",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "flate2"
version = "1.0.27"
@ -842,7 +939,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -1280,9 +1377,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.6.2"
version = "2.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5486aed0026218e61b8a01d5fbd5a0a134649abb71a0e53b7bc088529dced86e"
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
[[package]]
name = "memoffset"
@ -1329,7 +1426,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -1440,9 +1537,9 @@ dependencies = [
[[package]]
name = "object"
version = "0.32.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe"
checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
dependencies = [
"memchr",
]
@ -1670,7 +1767,7 @@ dependencies = [
"pest_meta",
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -1684,6 +1781,24 @@ dependencies = [
"sha2",
]
[[package]]
name = "phf"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
dependencies = [
"phf_shared",
]
[[package]]
name = "phf_shared"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
dependencies = [
"siphasher",
]
[[package]]
name = "pict-rs"
version = "0.5.0-alpha.17"
@ -1694,12 +1809,17 @@ dependencies = [
"actix-web",
"anyhow",
"async-trait",
"barrel",
"base64 0.21.3",
"clap",
"color-eyre",
"config",
"console-subscriber",
"dashmap",
"deadpool",
"diesel",
"diesel-async",
"diesel-derive-enum",
"flume",
"futures-core",
"hex",
@ -1713,6 +1833,7 @@ dependencies = [
"opentelemetry-otlp",
"pin-project-lite",
"quick-xml 0.30.0",
"refinery",
"reqwest",
"reqwest-middleware",
"reqwest-tracing",
@ -1728,13 +1849,13 @@ dependencies = [
"thiserror",
"time",
"tokio",
"tokio-postgres",
"tokio-uring",
"tokio-util",
"toml 0.7.6",
"tracing",
"tracing-actix-web",
"tracing-error",
"tracing-futures",
"tracing-log",
"tracing-opentelemetry",
"tracing-subscriber",
@ -1759,7 +1880,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -1780,6 +1901,53 @@ version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b"
[[package]]
name = "postgres"
version = "0.19.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7915b33ed60abc46040cbcaa25ffa1c7ec240668e0477c4f3070786f5916d451"
dependencies = [
"bytes",
"fallible-iterator",
"futures-util",
"log",
"tokio",
"tokio-postgres",
]
[[package]]
name = "postgres-protocol"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520"
dependencies = [
"base64 0.21.3",
"byteorder",
"bytes",
"fallible-iterator",
"hmac",
"md-5",
"memchr",
"rand",
"sha2",
"stringprep",
]
[[package]]
name = "postgres-types"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c"
dependencies = [
"bytes",
"fallible-iterator",
"postgres-protocol",
"serde",
"serde_json",
"time",
"uuid",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@ -1930,14 +2098,60 @@ dependencies = [
]
[[package]]
name = "regex"
version = "1.9.4"
name = "refinery"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
checksum = "cdb0436d0dd7bd8d4fce1e828751fa79742b08e35f27cfea7546f8a322b5ef24"
dependencies = [
"refinery-core",
"refinery-macros",
]
[[package]]
name = "refinery-core"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19206547cd047e8f4dfa6b20c30d3ecaf24be05841b6aa0aa926a47a3d0662bb"
dependencies = [
"async-trait",
"cfg-if",
"lazy_static",
"log",
"postgres",
"regex",
"serde",
"siphasher",
"thiserror",
"time",
"tokio",
"tokio-postgres",
"toml 0.7.6",
"url",
"walkdir",
]
[[package]]
name = "refinery-macros"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d94d4b9241859ba19eaa5c04c86e782eb3aa0aae2c5868e0cfa90c856e58a174"
dependencies = [
"proc-macro2",
"quote",
"refinery-core",
"regex",
"syn 2.0.31",
]
[[package]]
name = "regex"
version = "1.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.3.7",
"regex-automata 0.3.8",
"regex-syntax 0.7.5",
]
@ -1952,9 +2166,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.7"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
dependencies = [
"aho-corasick",
"memchr",
@ -2046,6 +2260,12 @@ dependencies = [
"tracing",
]
[[package]]
name = "retain_mut"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0"
[[package]]
name = "ring"
version = "0.16.20"
@ -2168,6 +2388,25 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "scoped-futures"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1473e24c637950c9bd38763220bea91ec3e095a89f672bbd7a10d03e77ba467"
dependencies = [
"cfg-if",
"pin-utils",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
@ -2232,7 +2471,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -2316,6 +2555,12 @@ dependencies = [
"libc",
]
[[package]]
name = "siphasher"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
[[package]]
name = "sketches-ddsketch"
version = "0.2.1"
@ -2395,6 +2640,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f11d35dae9818c4313649da4a97c8329e29357a7fe584526c1d78f5b63ef836"
[[package]]
name = "stringprep"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da"
dependencies = [
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "strsim"
version = "0.10.0"
@ -2420,9 +2675,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.29"
version = "2.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
dependencies = [
"proc-macro2",
"quote",
@ -2446,22 +2701,22 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.47"
version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.47"
version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -2555,7 +2810,33 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
name = "tokio-postgres"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8"
dependencies = [
"async-trait",
"byteorder",
"bytes",
"fallible-iterator",
"futures-channel",
"futures-util",
"log",
"parking_lot 0.12.1",
"percent-encoding",
"phf",
"pin-project-lite",
"postgres-protocol",
"postgres-types",
"rand",
"socket2 0.5.3",
"tokio",
"tokio-util",
"whoami",
]
[[package]]
@ -2727,8 +3008,7 @@ dependencies = [
[[package]]
name = "tracing-actix-web"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c0b08ce08cbde6a96fc1e4ebb8132053e53ec7a5cd27eef93ede6b73ebbda06"
source = "git+https://github.com/asonix/tracing-actix-web?branch=asonix/tracing-opentelemetry-021#0b337cc17fb88efc76d913751eee3ac66e4cd27f"
dependencies = [
"actix-web",
"opentelemetry",
@ -2746,7 +3026,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
]
[[package]]
@ -2769,16 +3049,6 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "tracing-futures"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
dependencies = [
"pin-project",
"tracing",
]
[[package]]
name = "tracing-log"
version = "0.1.3"
@ -2792,12 +3062,14 @@ dependencies = [
[[package]]
name = "tracing-opentelemetry"
version = "0.20.0"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc09e402904a5261e42cf27aea09ccb7d5318c6717a9eec3d8e2e65c56b18f19"
checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8"
dependencies = [
"once_cell",
"opentelemetry",
"opentelemetry_sdk",
"smallvec",
"tracing",
"tracing-core",
"tracing-log",
@ -2936,6 +3208,16 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "walkdir"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.1"
@ -2972,7 +3254,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
"wasm-bindgen-shared",
]
@ -3006,7 +3288,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.29",
"syn 2.0.31",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -3046,6 +3328,16 @@ version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
[[package]]
name = "whoami"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50"
dependencies = [
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "winapi"
version = "0.3.9"
@ -3062,6 +3354,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"

14
Cargo.toml
View File

@ -20,12 +20,17 @@ actix-server = "2.0.0"
actix-web = { version = "4.0.0", default-features = false }
anyhow = "1.0"
async-trait = "0.1.51"
barrel = { version = "0.7.0", features = ["pg"] }
base64 = "0.21.0"
clap = { version = "4.0.2", features = ["derive"] }
color-eyre = "0.6"
config = "0.13.0"
console-subscriber = "0.1"
dashmap = "5.1.0"
deadpool = { version = "0.9.5", features = ["rt_tokio_1"] }
diesel = { version = "2.1.1", features = ["postgres_backend", "serde_json", "time", "uuid"] }
diesel-async = { version = "0.4.1", features = ["postgres", "deadpool"] }
diesel-derive-enum = { version = "2.1.0", features = ["postgres"] }
flume = "0.11.0"
futures-core = "0.3"
hex = "0.4.3"
@ -39,6 +44,7 @@ opentelemetry = { version = "0.20", features = ["rt-tokio"] }
opentelemetry-otlp = "0.13"
pin-project-lite = "0.2.7"
quick-xml = { version = "0.30.0", features = ["serialize"] }
refinery = { version = "0.8.10", features = ["tokio-postgres", "postgres"] }
reqwest = { version = "0.11.18", default-features = false, features = ["json", "rustls-tls", "stream"] }
reqwest-middleware = "0.2.2"
reqwest-tracing = { version = "0.4.5" }
@ -54,6 +60,7 @@ storage-path-generator = "0.1.0"
thiserror = "1.0"
time = { version = "0.3.0", features = ["serde", "serde-well-known"] }
tokio = { version = "1", features = ["full", "tracing"] }
tokio-postgres = { version = "0.7.10", features = ["with-uuid-1", "with-time-0_3", "with-serde_json-1"] }
tokio-uring = { version = "0.4", optional = true, features = ["bytes"] }
tokio-util = { version = "0.7", default-features = false, features = [
"codec",
@ -62,9 +69,8 @@ tokio-util = { version = "0.7", default-features = false, features = [
toml = "0.7.0"
tracing = "0.1.15"
tracing-error = "0.2.0"
tracing-futures = "0.2.4"
tracing-log = "0.1.2"
tracing-opentelemetry = "0.20"
tracing-opentelemetry = "0.21"
tracing-subscriber = { version = "0.3.0", features = [
"ansi",
"env-filter",
@ -79,4 +85,6 @@ uuid = { version = "1", features = ["serde", "std", "v4", "v7"] }
[dependencies.tracing-actix-web]
version = "0.7.5"
default-features = false
features = ["opentelemetry_0_20"]
features = ["emit_event_on_error", "opentelemetry_0_20"]
git = "https://github.com/asonix/tracing-actix-web"
branch = "asonix/tracing-opentelemetry-021"

16
dev.toml
View File

@ -11,8 +11,12 @@ targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
buffer_capacity = 102400
[tracing.opentelemetry]
url = 'http://127.0.0.1:4317'
service_name = 'pict-rs'
targets = 'info'
targets = 'info,pict_rs=debug'
[metrics]
prometheus_address = "127.0.0.1:8070"
[old_repo]
path = 'data/sled-repo-local'
@ -59,10 +63,12 @@ crf_2160 = 15
crf_max = 12
[repo]
type = 'sled'
path = 'data/sled-repo-local'
cache_capacity = 67108864
export_path = "data/exports-local"
type = 'postgres'
url = 'postgres://pictrs:1234@localhost:5432/pictrs'
# [repo]
# type = 'sled'
# path = 'data/sled-repo-local'
[store]
type = 'filesystem'

37
docker/object-storage/docker-compose.yml
View File

@ -13,7 +13,7 @@ services:
# - "6669:6669"
# environment:
# - PICTRS__TRACING__CONSOLE__ADDRESS=0.0.0.0:6669
# - PICTRS__TRACING__OPENTELEMETRY__URL=http://otel:4137
# - PICTRS__TRACING__OPENTELEMETRY__URL=http://jaeger:4317
# - RUST_BACKTRACE=1
# stdin_open: true
# tty: true
@ -27,7 +27,7 @@ services:
# - "8081:8081"
# environment:
# - PICTRS_PROXY_UPSTREAM=http://pictrs:8080
# - PICTRS_PROXY_OPENTELEMETRY_URL=http://otel:4137
# - PICTRS_PROXY_OPENTELEMETRY_URL=http://jaeger:4317
minio:
image: quay.io/minio/minio
@ -39,7 +39,7 @@ services:
- ./storage/minio:/mnt
garage:
image: dxflrs/garage:v0.8.1
image: dxflrs/garage:v0.8.3
ports:
- "3900:3900"
- "3901:3901"
@ -47,26 +47,35 @@ services:
- "3903:3903"
- "3904:3904"
environment:
- RUST_LOG=debug
- RUST_LOG=info
volumes:
- ./storage/garage:/mnt
- ./garage.toml:/etc/garage.toml
otel:
image: otel/opentelemetry-collector:latest
command: --config otel-local-config.yaml
postgres:
image: postgres:15-alpine
ports:
- "5432:5432"
environment:
- PGDATA=/var/lib/postgresql/data
- POSTGRES_DB=pictrs
- POSTGRES_USER=pictrs
- POSTGRES_PASSWORD=1234
volumes:
- type: bind
source: ./otel.yml
target: /otel-local-config.yaml
restart: always
depends_on:
- jaeger
- ./storage/postgres:/var/lib/postgresql/data
jaeger:
image: jaegertracing/all-in-one:1
image: jaegertracing/all-in-one:1.48
ports:
- "6831:6831/udp"
- "6832:6832/udp"
- "5778:5778"
- "4317:4317"
- "4138:4138"
- "14250:14250"
- "14268:14268"
- "14269:14269"
- "9411:9411"
# To view traces, visit http://localhost:16686
- "16686:16686"
restart: always

16
docs/postgres-planning.md
View File

@ -88,13 +88,13 @@ methods:
```sql
CREATE TABLE aliases (
alias VARCHAR(30) PRIMARY KEY,
alias VARCHAR(50) PRIMARY KEY,
hash BYTEA NOT NULL REFERENCES hashes(hash) ON DELETE CASCADE,
delete_token VARCHAR(30) NOT NULL
);
CREATE INDEX alias_hashes_index ON aliases (hash);
CREATE INDEX aliases_hash_index ON aliases (hash);
```
@ -155,7 +155,7 @@ methods:
CREATE TYPE job_status AS ENUM ('new', 'running');
CREATE TABLE queue (
CREATE TABLE job_queue (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
queue VARCHAR(30) NOT NULL,
job JSONB NOT NULL,
@ -165,20 +165,20 @@ CREATE TABLE queue (
);
CREATE INDEX queue_status_index ON queue INCLUDE status;
CREATE INDEX heartbeat_index ON queue
CREATE INDEX queue_status_index ON queue INCLUDE queue, status;
CREATE INDEX heartbeat_index ON queue INCLUDE heartbeat;
```
claiming a job can be
```sql
UPDATE queue SET status = 'new', heartbeat = NULL
UPDATE job_queue SET status = 'new', heartbeat = NULL
WHERE
heartbeat IS NOT NULL AND heartbeat < NOW - INTERVAL '2 MINUTES';
UPDATE queue SET status = 'running', heartbeat = CURRENT_TIMESTAMP
UPDATE job_queue SET status = 'running', heartbeat = CURRENT_TIMESTAMP
WHERE id = (
SELECT id
FROM queue
FROM job_queue
WHERE status = 'new' AND queue = '$QUEUE'
ORDER BY queue_time ASC
FOR UPDATE SKIP LOCKED

1
flake.nix
View File

@ -32,6 +32,7 @@
cargo
cargo-outdated
clippy
diesel-cli
exiftool
ffmpeg_6-full
garage

7
scripts/update-schema.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
diesel \
--database-url 'postgres://pictrs:1234@localhost:5432/pictrs' \
print-schema \
--custom-type-derives "diesel::query_builder::QueryId" \
> src/repo/postgres/schema.rs

55
src/backgrounded.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use crate::{
error::Error,
repo::{ArcRepo, UploadId},
@ -9,19 +11,13 @@ use futures_core::Stream;
use mime::APPLICATION_OCTET_STREAM;
use tracing::{Instrument, Span};
pub(crate) struct Backgrounded<S>
where
S: Store,
{
pub(crate) struct Backgrounded {
repo: ArcRepo,
identifier: Option<S::Identifier>,
identifier: Option<Arc<str>>,
upload_id: Option<UploadId>,
}
impl<S> Backgrounded<S>
where
S: Store,
{
impl Backgrounded {
pub(crate) fn disarm(mut self) {
let _ = self.identifier.take();
let _ = self.upload_id.take();
@ -31,12 +27,13 @@ where
self.upload_id
}
pub(crate) fn identifier(&self) -> Option<&S::Identifier> {
pub(crate) fn identifier(&self) -> Option<&Arc<str>> {
self.identifier.as_ref()
}
pub(crate) async fn proxy<P>(repo: ArcRepo, store: S, stream: P) -> Result<Self, Error>
pub(crate) async fn proxy<S, P>(repo: ArcRepo, store: S, stream: P) -> Result<Self, Error>
where
S: Store,
P: Stream<Item = Result<Bytes, Error>> + Unpin + 'static,
{
let mut this = Self {
@ -50,8 +47,9 @@ where
Ok(this)
}
async fn do_proxy<P>(&mut self, store: S, stream: P) -> Result<(), Error>
async fn do_proxy<S, P>(&mut self, store: S, stream: P) -> Result<(), Error>
where
S: Store,
P: Stream<Item = Result<Bytes, Error>> + Unpin + 'static,
{
self.upload_id = Some(self.repo.create_upload().await?);
@ -68,10 +66,7 @@ where
}
}
impl<S> Drop for Backgrounded<S>
where
S: Store,
{
impl Drop for Backgrounded {
fn drop(&mut self) {
let any_items = self.identifier.is_some() || self.upload_id.is_some();
@ -87,14 +82,12 @@ where
let cleanup_span = tracing::info_span!(parent: &cleanup_parent_span, "Backgrounded cleanup Identifier", identifier = ?identifier);
tracing::trace_span!(parent: None, "Spawn task").in_scope(|| {
actix_rt::spawn(
async move {
let _ = crate::queue::cleanup_identifier(&repo, identifier).await;
}
.instrument(cleanup_span),
)
});
crate::sync::spawn(
async move {
let _ = crate::queue::cleanup_identifier(&repo, &identifier).await;
}
.instrument(cleanup_span),
);
}
if let Some(upload_id) = self.upload_id {
@ -102,14 +95,12 @@ where
let cleanup_span = tracing::info_span!(parent: &cleanup_parent_span, "Backgrounded cleanup Upload ID", upload_id = ?upload_id);
tracing::trace_span!(parent: None, "Spawn task").in_scope(|| {
actix_rt::spawn(
async move {
let _ = repo.claim(upload_id).await;
}
.instrument(cleanup_span),
)
});
crate::sync::spawn(
async move {
let _ = repo.claim(upload_id).await;
}
.instrument(cleanup_span),
);
}
}
}

4
src/config.rs
View File

@ -12,8 +12,8 @@ use defaults::Defaults;
pub(crate) use commandline::Operation;
pub(crate) use file::{
Animation, ConfigFile as Configuration, Image, Media, ObjectStorage, OpenTelemetry, Repo, Sled,
Store, Tracing, Video,
Animation, ConfigFile as Configuration, Image, Media, ObjectStorage, OpenTelemetry, Postgres,
Repo, Sled, Store, Tracing, Video,
};
pub(crate) use primitives::{Filesystem, LogFormat};

91
src/config/commandline.rs
View File

@ -369,8 +369,64 @@ impl Args {
from: from.into(),
to: to.into(),
},
config_file,
save_to,
config_file,
},
MigrateRepoTo::Postgres(MigratePostgresInner { to }) => Output {
config_format: ConfigFormat {
server,
client,
old_repo,
tracing,
metrics,
media,
repo: None,
store: None,
},
operation: Operation::MigrateRepo {
from: from.into(),
to: to.into(),
},
save_to,
config_file,
},
},
MigrateRepoFrom::Postgres(MigratePostgresRepo { from, to }) => match to {
MigrateRepoTo::Sled(MigrateSledInner { to }) => Output {
config_format: ConfigFormat {
server,
client,
old_repo,
tracing,
metrics,
media,
repo: None,
store: None,
},
operation: Operation::MigrateRepo {
from: from.into(),
to: to.into(),
},
save_to,
config_file,
},
MigrateRepoTo::Postgres(MigratePostgresInner { to }) => Output {
config_format: ConfigFormat {
server,
client,
old_repo,
tracing,
metrics,
media,
repo: None,
store: None,
},
operation: Operation::MigrateRepo {
from: from.into(),
to: to.into(),
},
save_to,
config_file,
},
},
}
@ -1058,6 +1114,7 @@ enum MigrateStoreFrom {
#[derive(Debug, Subcommand)]
enum MigrateRepoFrom {
Sled(MigrateSledRepo),
Postgres(MigratePostgresRepo),
}
/// Configure the destination storage for pict-rs storage migration
@ -1075,8 +1132,10 @@ enum MigrateStoreTo {
/// Configure the destination repo for pict-rs repo migration
#[derive(Debug, Subcommand)]
enum MigrateRepoTo {
/// Migrate to the provided sled storage
/// Migrate to the provided sled repo
Sled(MigrateSledInner),
/// Migrate to the provided postgres repo
Postgres(MigratePostgresInner),
}
/// Migrate pict-rs' storage from the provided filesystem storage
@ -1099,6 +1158,16 @@ struct MigrateSledRepo {
to: MigrateRepoTo,
}
/// Migrate pict-rs' repo from the provided postgres repo
#[derive(Debug, Parser)]
struct MigratePostgresRepo {
#[command(flatten)]
from: Postgres,
#[command(subcommand)]
to: MigrateRepoTo,
}
/// Migrate pict-rs' storage to the provided filesystem storage
#[derive(Debug, Parser)]
struct MigrateFilesystemInner {
@ -1116,6 +1185,13 @@ struct MigrateSledInner {
to: Sled,
}
/// Migrate pict-rs' repo to the provided postgres repo
#[derive(Debug, Parser)]
struct MigratePostgresInner {
#[command(flatten)]
to: Postgres,
}
/// Migrate pict-rs' storage from the provided object storage
#[derive(Debug, Parser)]
struct MigrateObjectStorage {
@ -1163,6 +1239,8 @@ struct RunObjectStorage {
enum Repo {
/// Run pict-rs with the provided sled-backed data repository
Sled(Sled),
/// Run pict-rs with the provided postgres-backed data repository
Postgres(Postgres),
}
/// Configuration for filesystem media storage
@ -1254,6 +1332,15 @@ pub(super) struct Sled {
pub(super) export_path: Option<PathBuf>,
}
/// Configuration for the postgres-backed data repository
#[derive(Debug, Parser, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(super) struct Postgres {
/// The URL of the postgres database
#[arg(short, long)]
pub(super) url: Url,
}
#[derive(Debug, Parser, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct OldSled {

12
src/config/defaults.rs
View File

@ -363,8 +363,20 @@ impl From<crate::config::commandline::Sled> for crate::config::file::Sled {
}
}
impl From<crate::config::commandline::Postgres> for crate::config::file::Postgres {
fn from(value: crate::config::commandline::Postgres) -> Self {
crate::config::file::Postgres { url: value.url }
}
}
impl From<crate::config::commandline::Sled> for crate::config::file::Repo {
fn from(value: crate::config::commandline::Sled) -> Self {
crate::config::file::Repo::Sled(value.into())
}
}
impl From<crate::config::commandline::Postgres> for crate::config::file::Repo {
fn from(value: crate::config::commandline::Postgres) -> Self {
crate::config::file::Repo::Postgres(value.into())
}
}

7
src/config/file.rs
View File

@ -88,6 +88,7 @@ pub(crate) struct ObjectStorage {
#[serde(tag = "type")]
pub(crate) enum Repo {
Sled(Sled),
Postgres(Postgres),
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
@ -421,3 +422,9 @@ pub(crate) struct Sled {
pub(crate) export_path: PathBuf,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct Postgres {
pub(crate) url: Url,
}

5
src/details.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use crate::{
bytes_stream::BytesStream,
discover::Discovery,
@ -101,9 +103,10 @@ impl Details {
))
}
#[tracing::instrument(level = "DEBUG")]
pub(crate) async fn from_store<S: Store>(
store: &S,
identifier: &S::Identifier,
identifier: &Arc<str>,
timeout: u64,
) -> Result<Self, Error> {
let mut buf = BytesStream::new();

1
src/discover/exiftool.rs
View File

@ -9,6 +9,7 @@ use crate::{
use super::Discovery;
#[tracing::instrument(level = "DEBUG", skip_all)]
pub(super) async fn check_reorient(
Discovery {
input,

2
src/discover/magick.rs
View File

@ -97,6 +97,7 @@ pub(super) async fn confirm_bytes(
.await
}
#[tracing::instrument(level = "DEBUG", skip(f))]
async fn count_avif_frames<F, Fut>(f: F, timeout: u64) -> Result<u32, MagickError>
where
F: FnOnce(crate::file::File) -> Fut,
@ -147,6 +148,7 @@ where
Ok(lines)
}
#[tracing::instrument(level = "DEBUG", skip(f))]
async fn discover_file<F, Fut>(f: F, timeout: u64) -> Result<Discovery, MagickError>
where
F: FnOnce(crate::file::File) -> Fut,

30
src/error.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use actix_web::{http::StatusCode, HttpResponse, ResponseError};
use color_eyre::Report;
@ -5,6 +7,8 @@ use crate::error_code::ErrorCode;
pub(crate) struct Error {
inner: color_eyre::Report,
debug: Arc<str>,
display: Arc<str>,
}
impl Error {
@ -21,17 +25,21 @@ impl Error {
.map(|e| e.error_code())
.unwrap_or(ErrorCode::UNKNOWN_ERROR)
}
pub(crate) fn is_disconnected(&self) -> bool {
self.kind().map(|e| e.is_disconnected()).unwrap_or(false)
}
}
impl std::fmt::Debug for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Debug::fmt(&self.inner, f)
f.write_str(&self.debug)
}
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.inner, f)
f.write_str(&self.display)
}
}
@ -46,8 +54,14 @@ where
UploadError: From<T>,
{
fn from(error: T) -> Self {
let inner = Report::from(UploadError::from(error));
let debug = Arc::from(format!("{inner:?}"));
let display = Arc::from(format!("{inner}"));
Error {
inner: Report::from(UploadError::from(error)),
inner,
debug,
display,
}
}
}
@ -166,12 +180,20 @@ impl UploadError {
Self::InvalidToken => ErrorCode::INVALID_DELETE_TOKEN,
Self::UnsupportedProcessExtension => ErrorCode::INVALID_FILE_EXTENSION,
Self::DuplicateAlias => ErrorCode::DUPLICATE_ALIAS,
Self::PushJob(_) => todo!(),
Self::PushJob(_) => ErrorCode::PUSH_JOB,
Self::Range => ErrorCode::RANGE_NOT_SATISFIABLE,
Self::Limit(_) => ErrorCode::VALIDATE_FILE_SIZE,
Self::Timeout(_) => ErrorCode::STREAM_TOO_SLOW,
}
}
const fn is_disconnected(&self) -> bool {
match self {
Self::Repo(e) => e.is_disconnected(),
Self::Store(s) => s.is_disconnected(),
_ => false,
}
}
}
impl From<actix_web::error::BlockingError> for UploadError {

7
src/error_code.rs
View File

@ -56,12 +56,19 @@ impl ErrorCode {
code: "already-claimed",
};
pub(crate) const SLED_ERROR: ErrorCode = ErrorCode { code: "sled-error" };
pub(crate) const POSTGRES_ERROR: ErrorCode = ErrorCode {
code: "postgres-error",
};
pub(crate) const EXTRACT_DETAILS: ErrorCode = ErrorCode {
code: "extract-details",
};
pub(crate) const EXTRACT_UPLOAD_RESULT: ErrorCode = ErrorCode {
code: "extract-upload-result",
};
pub(crate) const PUSH_JOB: ErrorCode = ErrorCode { code: "push-job" };
pub(crate) const EXTRACT_JOB: ErrorCode = ErrorCode {
code: "extract-job",
};
pub(crate) const CONFLICTED_RECORD: ErrorCode = ErrorCode {
code: "conflicted-record",
};

4
src/ffmpeg.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use crate::{
error_code::ErrorCode,
formats::InternalVideoFormat,
@ -132,7 +134,7 @@ impl ThumbnailFormat {
#[tracing::instrument(skip(store))]
pub(crate) async fn thumbnail<S: Store>(
store: S,
from: S::Identifier,
from: Arc<str>,
input_format: InternalVideoFormat,
format: ThumbnailFormat,
timeout: u64,

6
src/file.rs
View File

@ -446,15 +446,15 @@ mod io_uring {
actix_rt::System::new().block_on(async move {
let arbiter = actix_rt::Arbiter::new();
let (tx, rx) = tokio::sync::oneshot::channel();
let (tx, rx) = crate::sync::channel(1);
arbiter.spawn(async move {
let handle = actix_rt::spawn($fut);
let handle = crate::sync::spawn($fut);
let _ = tx.send(handle.await.unwrap());
});
rx.await.unwrap()
rx.into_recv_async().await.unwrap()
})
};
}

75
src/future.rs Normal file
View File

@ -0,0 +1,75 @@
use std::{
future::Future,
time::{Duration, Instant},
};
pub(crate) type LocalBoxFuture<'a, T> = std::pin::Pin<Box<dyn Future<Output = T> + 'a>>;
pub(crate) trait WithTimeout: Future {
fn with_timeout(self, duration: Duration) -> actix_rt::time::Timeout<Self>
where
Self: Sized,
{
actix_rt::time::timeout(duration, self)
}
}
pub(crate) trait WithMetrics: Future {
fn with_metrics(self, name: &'static str) -> MetricsFuture<Self>
where
Self: Sized,
{
MetricsFuture {
future: self,
metrics: Metrics {
name,
start: Instant::now(),
complete: false,
},
}
}
}
impl<F> WithMetrics for F where F: Future {}
impl<F> WithTimeout for F where F: Future {}
pin_project_lite::pin_project! {
pub(crate) struct MetricsFuture<F> {
#[pin]
future: F,
metrics: Metrics,
}
}
struct Metrics {
name: &'static str,
start: Instant,
complete: bool,
}
impl<F> Future for MetricsFuture<F>
where
F: Future,
{
type Output = F::Output;
fn poll(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Self::Output> {
let this = self.project();
let out = std::task::ready!(this.future.poll(cx));
this.metrics.complete = true;
std::task::Poll::Ready(out)
}
}
impl Drop for Metrics {
fn drop(&mut self) {
metrics::histogram!(self.name, self.start.elapsed().as_secs_f64(), "complete" => self.complete.to_string());
}
}

6
src/generate.rs
View File

@ -5,7 +5,7 @@ use crate::{
ffmpeg::ThumbnailFormat,
formats::{InputProcessableFormat, InternalVideoFormat},
repo::{Alias, ArcRepo, Hash, VariantAlreadyExists},
store::{Identifier, Store},
store::Store,
};
use actix_web::web::Bytes;
use std::{path::PathBuf, time::Instant};
@ -91,7 +91,7 @@ async fn process<S: Store + 'static>(
let permit = crate::PROCESS_SEMAPHORE.acquire().await;
let identifier = if let Some(identifier) = repo.still_identifier_from_alias(&alias).await? {
S::Identifier::from_arc(identifier)?
identifier
} else {
let Some(identifier) = repo.identifier(hash.clone()).await? else {
return Err(UploadError::MissingIdentifier.into());
@ -101,7 +101,7 @@ async fn process<S: Store + 'static>(
let reader = crate::ffmpeg::thumbnail(
store.clone(),
S::Identifier::from_arc(identifier)?,
identifier,
input_format.unwrap_or(InternalVideoFormat::Mp4),
thumbnail_format,
media.process_timeout,

67
src/ingest.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use crate::{
bytes_stream::BytesStream,
either::Either,
@ -15,15 +17,12 @@ mod hasher;
use hasher::Hasher;
#[derive(Debug)]
pub(crate) struct Session<S>
where
S: Store,
{
pub(crate) struct Session {
repo: ArcRepo,
delete_token: DeleteToken,
hash: Option<Hash>,
alias: Option<Alias>,
identifier: Option<S::Identifier>,
identifier: Option<Arc<str>>,
}
#[tracing::instrument(skip(stream))]
@ -49,7 +48,7 @@ pub(crate) async fn ingest<S>(
stream: impl Stream<Item = Result<Bytes, Error>> + Unpin + 'static,
declared_alias: Option<Alias>,
media: &crate::config::Media,
) -> Result<Session<S>, Error>
) -> Result<Session, Error>
where
S: Store,
{
@ -131,11 +130,11 @@ where
#[tracing::instrument(level = "trace", skip_all)]
async fn save_upload<S>(
session: &mut Session<S>,
session: &mut Session,
repo: &ArcRepo,
store: &S,
hash: Hash,
identifier: &S::Identifier,
identifier: &Arc<str>,
) -> Result<(), Error>
where
S: Store,
@ -153,10 +152,7 @@ where
Ok(())
}
impl<S> Session<S>
where
S: Store,
{
impl Session {
pub(crate) fn disarm(mut self) -> DeleteToken {
let _ = self.hash.take();
let _ = self.alias.take();
@ -206,10 +202,7 @@ where
}
}
impl<S> Drop for Session<S>
where
S: Store,
{
impl Drop for Session {
fn drop(&mut self) {
let any_items = self.hash.is_some() || self.alias.is_some() || self.identifier.is_some();
@ -224,14 +217,12 @@ where
let cleanup_span = tracing::info_span!(parent: &cleanup_parent_span, "Session cleanup hash", hash = ?hash);
tracing::trace_span!(parent: None, "Spawn task").in_scope(|| {
actix_rt::spawn(
async move {
let _ = crate::queue::cleanup_hash(&repo, hash).await;
}
.instrument(cleanup_span),
)
});
crate::sync::spawn(
async move {
let _ = crate::queue::cleanup_hash(&repo, hash).await;
}
.instrument(cleanup_span),
);
}
if let Some(alias) = self.alias.take() {
@ -240,14 +231,12 @@ where
let cleanup_span = tracing::info_span!(parent: &cleanup_parent_span, "Session cleanup alias", alias = ?alias);
tracing::trace_span!(parent: None, "Spawn task").in_scope(|| {
actix_rt::spawn(
async move {
let _ = crate::queue::cleanup_alias(&repo, alias, token).await;
}
.instrument(cleanup_span),
)
});
crate::sync::spawn(
async move {
let _ = crate::queue::cleanup_alias(&repo, alias, token).await;
}
.instrument(cleanup_span),
);
}
if let Some(identifier) = self.identifier.take() {
@ -255,14 +244,12 @@ where
let cleanup_span = tracing::info_span!(parent: &cleanup_parent_span, "Session cleanup identifier", identifier = ?identifier);
tracing::trace_span!(parent: None, "Spawn task").in_scope(|| {
actix_rt::spawn(
async move {
let _ = crate::queue::cleanup_identifier(&repo, identifier).await;
}
.instrument(cleanup_span),
)
});
crate::sync::spawn(
async move {
let _ = crate::queue::cleanup_identifier(&repo, &identifier).await;
}
.instrument(cleanup_span),
);
}
}
}

7
src/ingest/hasher.rs
View File

@ -82,16 +82,15 @@ mod test {
actix_rt::System::new().block_on(async move {
let arbiter = actix_rt::Arbiter::new();
let (tx, rx) = tracing::trace_span!(parent: None, "Create channel")
.in_scope(|| tokio::sync::oneshot::channel());
let (tx, rx) = crate::sync::channel(1);
arbiter.spawn(async move {
let handle = actix_rt::spawn($fut);
let handle = crate::sync::spawn($fut);
let _ = tx.send(handle.await.unwrap());
});
rx.await.unwrap()
rx.into_recv_async().await.unwrap()
})
};
}

7
src/init_tracing.rs
View File

@ -8,9 +8,7 @@ use opentelemetry_otlp::WithExportConfig;
use tracing::subscriber::set_global_default;
use tracing_error::ErrorLayer;
use tracing_log::LogTracer;
use tracing_subscriber::{
fmt::format::FmtSpan, layer::SubscriberExt, registry::LookupSpan, Layer, Registry,
};
use tracing_subscriber::{layer::SubscriberExt, registry::LookupSpan, Layer, Registry};
pub(super) fn init_tracing(tracing: &Tracing) -> color_eyre::Result<()> {
color_eyre::install()?;
@ -19,8 +17,7 @@ pub(super) fn init_tracing(tracing: &Tracing) -> color_eyre::Result<()> {
opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
let format_layer =
tracing_subscriber::fmt::layer().with_span_events(FmtSpan::NEW | FmtSpan::CLOSE);
let format_layer = tracing_subscriber::fmt::layer();
match tracing.logging.format {
LogFormat::Compact => with_format(format_layer.compact(), tracing),

192
src/lib.rs
View File

@ -11,6 +11,7 @@ mod exiftool;
mod ffmpeg;
mod file;
mod formats;
mod future;
mod generate;
mod ingest;
mod init_tracing;
@ -26,6 +27,7 @@ mod repo_04;
mod serde_str;
mod store;
mod stream;
mod sync;
mod tmp_file;
mod validate;
@ -36,6 +38,7 @@ use actix_web::{
web, App, HttpRequest, HttpResponse, HttpResponseBuilder, HttpServer,
};
use details::{ApiDetails, HumanDate};
use future::WithTimeout;
use futures_core::Stream;
use metrics_exporter_prometheus::PrometheusBuilder;
use middleware::Metrics;
@ -45,14 +48,15 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_tracing::TracingMiddleware;
use rusty_s3::UrlStyle;
use std::{
marker::PhantomData,
path::Path,
path::PathBuf,
sync::Arc,
time::{Duration, SystemTime},
};
use tokio::sync::Semaphore;
use tracing::Instrument;
use tracing_actix_web::TracingLogger;
use tracing_futures::Instrument;
use self::{
backgrounded::Backgrounded,
@ -69,7 +73,7 @@ use self::{
queue::queue_generate,
repo::{sled::SledRepo, Alias, DeleteToken, Hash, Repo, UploadId, UploadResult},
serde_str::Serde,
store::{file_store::FileStore, object_store::ObjectStore, Identifier, Store},
store::{file_store::FileStore, object_store::ObjectStore, Store},
stream::{empty, once, StreamLimit, StreamMap, StreamTimeout},
};
@ -83,8 +87,9 @@ const DAYS: u32 = 24 * HOURS;
const NOT_FOUND_KEY: &str = "404-alias";
static PROCESS_SEMAPHORE: Lazy<Semaphore> = Lazy::new(|| {
tracing::trace_span!(parent: None, "Initialize semaphore")
.in_scope(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)))
let permits = num_cpus::get().saturating_sub(1).max(1);
crate::sync::bare_semaphore(permits)
});
async fn ensure_details<S: Store + 'static>(
@ -93,7 +98,7 @@ async fn ensure_details<S: Store + 'static>(
config: &Configuration,
alias: &Alias,
) -> Result<Details, Error> {
let Some(identifier) = repo.identifier_from_alias(alias).await?.map(S::Identifier::from_arc).transpose()? else {
let Some(identifier) = repo.identifier_from_alias(alias).await? else {
return Err(UploadError::MissingAlias.into());
};
@ -117,10 +122,10 @@ async fn ensure_details<S: Store + 'static>(
}
}
struct Upload<S: Store + 'static>(Value<Session<S>>);
struct Upload<S>(Value<Session>, PhantomData<S>);
impl<S: Store + 'static> FormData for Upload<S> {
type Item = Session<S>;
type Item = Session;
type Error = Error;
fn form(req: &HttpRequest) -> Form<Self::Item, Self::Error> {
@ -172,14 +177,14 @@ impl<S: Store + 'static> FormData for Upload<S> {
}
fn extract(value: Value<Self::Item>) -> Result<Self, Self::Error> {
Ok(Upload(value))
Ok(Upload(value, PhantomData))
}
}
struct Import<S: Store + 'static>(Value<Session<S>>);
struct Import<S: Store + 'static>(Value<Session>, PhantomData<S>);
impl<S: Store + 'static> FormData for Import<S> {
type Item = Session<S>;
type Item = Session;
type Error = Error;
fn form(req: &actix_web::HttpRequest) -> Form<Self::Item, Self::Error> {
@ -241,14 +246,14 @@ impl<S: Store + 'static> FormData for Import<S> {
where
Self: Sized,
{
Ok(Import(value))
Ok(Import(value, PhantomData))
}
}
/// Handle responding to successful uploads
#[tracing::instrument(name = "Uploaded files", skip(value, repo, store, config))]
async fn upload<S: Store + 'static>(
Multipart(Upload(value)): Multipart<Upload<S>>,
Multipart(Upload(value, _)): Multipart<Upload<S>>,
repo: web::Data<ArcRepo>,
store: web::Data<S>,
config: web::Data<Configuration>,
@ -259,7 +264,7 @@ async fn upload<S: Store + 'static>(
/// Handle responding to successful uploads
#[tracing::instrument(name = "Imported files", skip(value, repo, store, config))]
async fn import<S: Store + 'static>(
Multipart(Import(value)): Multipart<Import<S>>,
Multipart(Import(value, _)): Multipart<Import<S>>,
repo: web::Data<ArcRepo>,
store: web::Data<S>,
config: web::Data<Configuration>,
@ -270,7 +275,7 @@ async fn import<S: Store + 'static>(
/// Handle responding to successful uploads
#[tracing::instrument(name = "Uploaded files", skip(value, repo, store, config))]
async fn handle_upload<S: Store + 'static>(
value: Value<Session<S>>,
value: Value<Session>,
repo: web::Data<ArcRepo>,
store: web::Data<S>,
config: web::Data<Configuration>,
@ -312,10 +317,10 @@ async fn handle_upload<S: Store + 'static>(
})))
}
struct BackgroundedUpload<S: Store + 'static>(Value<Backgrounded<S>>);
struct BackgroundedUpload<S: Store + 'static>(Value<Backgrounded>, PhantomData<S>);
impl<S: Store + 'static> FormData for BackgroundedUpload<S> {
type Item = Backgrounded<S>;
type Item = Backgrounded;
type Error = Error;
fn form(req: &actix_web::HttpRequest) -> Form<Self::Item, Self::Error> {
@ -371,13 +376,13 @@ impl<S: Store + 'static> FormData for BackgroundedUpload<S> {
where
Self: Sized,
{
Ok(BackgroundedUpload(value))
Ok(BackgroundedUpload(value, PhantomData))
}
}
#[tracing::instrument(name = "Uploaded files", skip(value, repo))]
async fn upload_backgrounded<S: Store>(
Multipart(BackgroundedUpload(value)): Multipart<BackgroundedUpload<S>>,
Multipart(BackgroundedUpload(value, _)): Multipart<BackgroundedUpload<S>>,
repo: web::Data<ArcRepo>,
) -> Result<HttpResponse, Error> {
let images = value
@ -394,11 +399,7 @@ async fn upload_backgrounded<S: Store>(
for image in &images {
let upload_id = image.result.upload_id().expect("Upload ID exists");
let identifier = image
.result
.identifier()
.expect("Identifier exists")
.to_bytes()?;
let identifier = image.result.identifier().expect("Identifier exists");
queue::queue_ingest(&repo, identifier, upload_id, None).await?;
@ -432,7 +433,11 @@ async fn claim_upload<S: Store + 'static>(
) -> Result<HttpResponse, Error> {
let upload_id = Serde::into_inner(query.into_inner().upload_id);
match actix_rt::time::timeout(Duration::from_secs(10), repo.wait(upload_id)).await {
match repo
.wait(upload_id)
.with_timeout(Duration::from_secs(10))
.await
{
Ok(wait_res) => {
let upload_result = wait_res?;
repo.claim(upload_id).await?;
@ -560,10 +565,7 @@ async fn do_download_backgrounded<S: Store + 'static>(
let backgrounded = Backgrounded::proxy((**repo).clone(), (**store).clone(), stream).await?;
let upload_id = backgrounded.upload_id().expect("Upload ID exists");
let identifier = backgrounded
.identifier()
.expect("Identifier exists")
.to_bytes()?;
let identifier = backgrounded.identifier().expect("Identifier exists");
queue::queue_ingest(&repo, identifier, upload_id, None).await?;
@ -764,8 +766,6 @@ async fn process_details<S: Store>(
let identifier = repo
.variant_identifier(hash, thumbnail_string)
.await?
.map(S::Identifier::from_arc)
.transpose()?
.ok_or(UploadError::MissingAlias)?;
let details = repo.details(&identifier).await?;
@ -856,11 +856,7 @@ async fn process<S: Store + 'static>(
.await?;
}
let identifier_opt = repo
.variant_identifier(hash.clone(), path_string)
.await?
.map(S::Identifier::from_arc)
.transpose()?;
let identifier_opt = repo.variant_identifier(hash.clone(), path_string).await?;
if let Some(identifier) = identifier_opt {
let details = repo.details(&identifier).await?;
@ -980,11 +976,7 @@ async fn process_head<S: Store + 'static>(
.await?;
}
let identifier_opt = repo
.variant_identifier(hash.clone(), path_string)
.await?
.map(S::Identifier::from_arc)
.transpose()?;
let identifier_opt = repo.variant_identifier(hash.clone(), path_string).await?;
if let Some(identifier) = identifier_opt {
let details = repo.details(&identifier).await?;
@ -1047,11 +1039,7 @@ async fn process_backgrounded<S: Store>(
return Ok(HttpResponse::BadRequest().finish());
};
let identifier_opt = repo
.variant_identifier(hash.clone(), path_string)
.await?
.map(S::Identifier::from_arc)
.transpose()?;
let identifier_opt = repo.variant_identifier(hash.clone(), path_string).await?;
if identifier_opt.is_some() {
return Ok(HttpResponse::Accepted().finish());
@ -1185,7 +1173,7 @@ async fn do_serve<S: Store + 'static>(
(hash, alias, true)
};
let Some(identifier) = repo.identifier(hash.clone()).await?.map(Identifier::from_arc).transpose()? else {
let Some(identifier) = repo.identifier(hash.clone()).await? else {
tracing::warn!(
"Original File identifier for hash {hash:?} is missing, queue cleanup task",
);
@ -1250,7 +1238,7 @@ async fn do_serve_head<S: Store + 'static>(
store: web::Data<S>,
config: web::Data<Configuration>,
) -> Result<HttpResponse, Error> {
let Some(identifier) = repo.identifier_from_alias(&alias).await?.map(S::Identifier::from_arc).transpose()? else {
let Some(identifier) = repo.identifier_from_alias(&alias).await? else {
// Invalid alias
return Ok(HttpResponse::NotFound().finish());
};
@ -1268,7 +1256,7 @@ async fn do_serve_head<S: Store + 'static>(
async fn ranged_file_head_resp<S: Store + 'static>(
store: &S,
identifier: S::Identifier,
identifier: Arc<str>,
range: Option<web::Header<Range>>,
details: Details,
) -> Result<HttpResponse, Error> {
@ -1303,7 +1291,7 @@ async fn ranged_file_head_resp<S: Store + 'static>(
async fn ranged_file_resp<S: Store + 'static>(
store: &S,
identifier: S::Identifier,
identifier: Arc<str>,
range: Option<web::Header<Range>>,
details: Details,
not_found: bool,
@ -1555,7 +1543,7 @@ async fn identifier<S: Store>(
}
};
let Some(identifier) = repo.identifier_from_alias(&alias).await?.map(S::Identifier::from_arc).transpose()? else {
let Some(identifier) = repo.identifier_from_alias(&alias).await? else {
// Invalid alias
return Ok(HttpResponse::NotFound().json(serde_json::json!({
"msg": "No identifiers associated with provided alias"
@ -1564,10 +1552,11 @@ async fn identifier<S: Store>(
Ok(HttpResponse::Ok().json(&serde_json::json!({
"msg": "ok",
"identifier": identifier.string_repr(),
"identifier": identifier.as_ref(),
})))
}
#[tracing::instrument(skip(repo, store))]
async fn healthz<S: Store>(
repo: web::Data<ArcRepo>,
store: web::Data<S>,
@ -1691,44 +1680,39 @@ fn spawn_cleanup(repo: ArcRepo, config: &Configuration) {
return;
}
tracing::trace_span!(parent: None, "Spawn task").in_scope(|| {
actix_rt::spawn(async move {
let mut interval = actix_rt::time::interval(Duration::from_secs(30));
crate::sync::spawn(async move {
let mut interval = actix_rt::time::interval(Duration::from_secs(30));
loop {
interval.tick().await;
loop {
interval.tick().await;
if let Err(e) = queue::cleanup_outdated_variants(&repo).await {
tracing::warn!(
"Failed to spawn cleanup for outdated variants:{}",
format!("\n{e}\n{e:?}")
);
}
if let Err(e) = queue::cleanup_outdated_proxies(&repo).await {
tracing::warn!(
"Failed to spawn cleanup for outdated proxies:{}",
format!("\n{e}\n{e:?}")
);
}
if let Err(e) = queue::cleanup_outdated_variants(&repo).await {
tracing::warn!(
"Failed to spawn cleanup for outdated variants:{}",
format!("\n{e}\n{e:?}")
);
}
});
})
if let Err(e) = queue::cleanup_outdated_proxies(&repo).await {
tracing::warn!(
"Failed to spawn cleanup for outdated proxies:{}",
format!("\n{e}\n{e:?}")
);
}
}
});
}
fn spawn_workers<S>(repo: ArcRepo, store: S, config: Configuration, process_map: ProcessMap)
where
S: Store + 'static,
{
tracing::trace_span!(parent: None, "Spawn task").in_scope(|| {
actix_rt::spawn(queue::process_cleanup(
repo.clone(),
store.clone(),
config.clone(),
))
});
tracing::trace_span!(parent: None, "Spawn task")
.in_scope(|| actix_rt::spawn(queue::process_images(repo, store, process_map, config)));
crate::sync::spawn(queue::process_cleanup(
repo.clone(),
store.clone(),
config.clone(),
));
crate::sync::spawn(queue::process_images(repo, store, process_map, config));
}
async fn launch_file_store<F: Fn(&mut web::ServiceConfig) + Send + Clone + 'static>(
@ -1810,7 +1794,7 @@ async fn launch_object_store<F: Fn(&mut web::ServiceConfig) + Send + Clone + 'st
}
async fn migrate_inner<S1>(
repo: Repo,
repo: ArcRepo,
client: ClientWithMiddleware,
from: S1,
to: config::primitives::Store,
@ -1824,11 +1808,7 @@ where
config::primitives::Store::Filesystem(config::Filesystem { path }) => {
let to = FileStore::build(path.clone(), repo.clone()).await?;
match repo {
Repo::Sled(repo) => {
migrate_store(Arc::new(repo), from, to, skip_missing_files, timeout).await?
}
}
migrate_store(repo, from, to, skip_missing_files, timeout).await?
}
config::primitives::Store::ObjectStorage(config::primitives::ObjectStorage {
endpoint,
@ -1862,11 +1842,7 @@ where
.await?
.build(client);
match repo {
Repo::Sled(repo) => {
migrate_store(Arc::new(repo), from, to, skip_missing_files, timeout).await?
}
}
migrate_store(repo, from, to, skip_missing_files, timeout).await?
}
}
@ -1970,7 +1946,7 @@ impl PictRsConfiguration {
from,
to,
} => {
let repo = Repo::open(config.repo.clone())?;
let repo = Repo::open(config.repo.clone()).await?.to_arc();
match from {
config::primitives::Store::Filesystem(config::Filesystem { path }) => {
@ -2034,15 +2010,15 @@ impl PictRsConfiguration {
return Ok(());
}
Operation::MigrateRepo { from, to } => {
let from = Repo::open(from)?.to_arc();
let to = Repo::open(to)?.to_arc();
let from = Repo::open(from).await?.to_arc();
let to = Repo::open(to).await?.to_arc();
repo::migrate_repo(from, to).await?;
return Ok(());
}
}
let repo = Repo::open(config.repo.clone())?;
let repo = Repo::open(config.repo.clone()).await?;
if config.server.read_only {
tracing::warn!("Launching in READ ONLY mode");
@ -2050,10 +2026,10 @@ impl PictRsConfiguration {
match config.store.clone() {
config::Store::Filesystem(config::Filesystem { path }) => {
let store = FileStore::build(path, repo.clone()).await?;
let arc_repo = repo.to_arc();
let store = FileStore::build(path, arc_repo.clone()).await?;
if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone())
@ -2066,15 +2042,14 @@ impl PictRsConfiguration {
match repo {
Repo::Sled(sled_repo) => {
launch_file_store(
Arc::new(sled_repo.clone()),
store,
client,
config,
move |sc| sled_extra_config(sc, sled_repo.clone()),
)
launch_file_store(arc_repo, store, client, config, move |sc| {
sled_extra_config(sc, sled_repo.clone())
})
.await?;
}
Repo::Postgres(_) => {
launch_file_store(arc_repo, store, client, config, |_| {}).await?;
}
}
}
config::Store::ObjectStorage(config::ObjectStorage {
@ -2089,6 +2064,8 @@ impl PictRsConfiguration {
client_timeout,
public_endpoint,
}) => {
let arc_repo = repo.to_arc();
let store = ObjectStore::build(
endpoint,
bucket_name,
@ -2104,13 +2081,11 @@ impl PictRsConfiguration {
signature_duration,
client_timeout,
public_endpoint,
repo.clone(),
arc_repo.clone(),
)
.await?
.build(client.clone());
let arc_repo = repo.to_arc();
if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone())
@ -2128,6 +2103,9 @@ impl PictRsConfiguration {
})
.await?;
}
Repo::Postgres(_) => {
launch_object_store(arc_repo, store, client, config, |_| {}).await?;
}
}
}
}

4
src/magick.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use crate::{
error_code::ErrorCode,
formats::ProcessableFormat,
@ -140,7 +142,7 @@ where
pub(crate) async fn process_image_store_read<S: Store + 'static>(
store: &S,
identifier: &S::Identifier,
identifier: &Arc<str>,
args: Vec<String>,
input_format: ProcessableFormat,
format: ProcessableFormat,

12
src/middleware.rs
View File

@ -12,6 +12,8 @@ use std::{
task::{Context, Poll},
};
use crate::future::WithTimeout;
pub(crate) use self::metrics::Metrics;
pub(crate) struct Deadline;
@ -149,8 +151,12 @@ impl actix_web::error::ResponseError for DeadlineExceeded {
HttpResponse::build(self.status_code())
.content_type("application/json")
.body(
serde_json::to_string(&serde_json::json!({ "msg": self.to_string() }))
.unwrap_or_else(|_| r#"{"msg":"request timeout"}"#.to_string()),
serde_json::to_string(
&serde_json::json!({ "msg": self.to_string(), "code": "request-timeout" }),
)
.unwrap_or_else(|_| {
r#"{"msg":"request timeout","code":"request-timeout"}"#.to_string()
}),
)
}
}
@ -163,7 +169,7 @@ where
DeadlineFuture {
inner: match timeout {
Some(duration) => DeadlineFutureInner::Timed {
timeout: actix_rt::time::timeout(duration, future),
timeout: future.with_timeout(duration),
},
None => DeadlineFutureInner::Untimed { future },
},

33
src/migrate_store.rs
View File

@ -1,6 +1,9 @@
use std::{
rc::Rc,
sync::atomic::{AtomicU64, Ordering},
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
time::{Duration, Instant},
};
@ -8,7 +11,7 @@ use crate::{
details::Details,
error::{Error, UploadError},
repo::{ArcRepo, Hash},
store::{Identifier, Store},
store::Store,
stream::IntoStreamer,
};
@ -58,7 +61,7 @@ where
tracing::warn!("Retrying migration +{failure_count}");
}
tokio::time::sleep(Duration::from_secs(3)).await;
actix_rt::time::sleep(Duration::from_secs(3)).await;
}
Ok(())
@ -103,7 +106,7 @@ where
}
// Hashes are read in a consistent order
let mut stream = repo.hashes().await.into_streamer();
let mut stream = repo.hashes().into_streamer();
let state = Rc::new(MigrateState {
repo: repo.clone(),
@ -169,7 +172,7 @@ where
let current_index = index.fetch_add(1, Ordering::Relaxed);
let original_identifier = match repo.identifier(hash.clone()).await {
Ok(Some(identifier)) => S1::Identifier::from_arc(identifier)?,
Ok(Some(identifier)) => identifier,
Ok(None) => {
tracing::warn!(
"Original File identifier for hash {hash:?} is missing, queue cleanup task",
@ -214,8 +217,6 @@ where
}
if let Some(identifier) = repo.motion_identifier(hash.clone()).await? {
let identifier = S1::Identifier::from_arc(identifier)?;
if !repo.is_migrated(&identifier).await? {
match migrate_file(repo, from, to, &identifier, *skip_missing_files, *timeout).await {
Ok(new_identifier) => {
@ -245,8 +246,6 @@ where
}
for (variant, identifier) in repo.variants(hash.clone()).await? {
let identifier = S1::Identifier::from_arc(identifier)?;
if !repo.is_migrated(&identifier).await? {
match migrate_file(repo, from, to, &identifier, *skip_missing_files, *timeout).await {
Ok(new_identifier) => {
@ -339,10 +338,10 @@ async fn migrate_file<S1, S2>(
repo: &ArcRepo,
from: &S1,
to: &S2,
identifier: &S1::Identifier,
identifier: &Arc<str>,
skip_missing_files: bool,
timeout: u64,
) -> Result<S2::Identifier, MigrateError>
) -> Result<Arc<str>, MigrateError>
where
S1: Store,
S2: Store,
@ -365,7 +364,7 @@ where
tracing::warn!("Failed moving file. Retrying +{failure_count}");
}
tokio::time::sleep(Duration::from_secs(3)).await;
actix_rt::time::sleep(Duration::from_secs(3)).await;
}
}
}
@ -382,9 +381,9 @@ async fn do_migrate_file<S1, S2>(
repo: &ArcRepo,
from: &S1,
to: &S2,
identifier: &S1::Identifier,
identifier: &Arc<str>,
timeout: u64,
) -> Result<S2::Identifier, MigrateError>
) -> Result<Arc<str>, MigrateError>
where
S1: Store,
S2: Store,
@ -421,11 +420,7 @@ where
Ok(new_identifier)
}
async fn migrate_details<I1, I2>(repo: &ArcRepo, from: &I1, to: &I2) -> Result<(), Error>
where
I1: Identifier,
I2: Identifier,
{
async fn migrate_details(repo: &ArcRepo, from: &Arc<str>, to: &Arc<str>) -> Result<(), Error> {
if let Some(details) = repo.details(from).await? {
repo.relate_details(to, &details).await?;
repo.cleanup_details(from).await?;

58
src/process.rs
View File

@ -1,5 +1,6 @@
use actix_rt::task::JoinHandle;
use actix_web::web::Bytes;
use flume::r#async::RecvFut;
use std::{
future::Future,
pin::Pin,
@ -10,11 +11,10 @@ use std::{
use tokio::{
io::{AsyncRead, AsyncWriteExt, ReadBuf},
process::{Child, ChildStdin, ChildStdout, Command},
sync::oneshot::{channel, Receiver},
};
use tracing::{Instrument, Span};
use crate::error_code::ErrorCode;
use crate::{error_code::ErrorCode, future::WithTimeout};
struct MetricsGuard {
start: Instant,
@ -73,7 +73,7 @@ struct DropHandle {
pub(crate) struct ProcessRead<I> {
inner: I,
err_recv: Receiver<std::io::Error>,
err_recv: RecvFut<'static, std::io::Error>,
err_closed: bool,
#[allow(dead_code)]
handle: DropHandle,
@ -159,7 +159,7 @@ impl Process {
timeout,
} = self;
let res = actix_rt::time::timeout(timeout, child.wait()).await;
let res = child.wait().with_timeout(timeout).await;
match res {
Ok(Ok(status)) if status.success() => {
@ -206,39 +206,37 @@ impl Process {
let stdin = child.stdin.take().expect("stdin exists");
let stdout = child.stdout.take().expect("stdout exists");
let (tx, rx) = tracing::trace_span!(parent: None, "Create channel", %command)
.in_scope(channel::<std::io::Error>);
let (tx, rx) = crate::sync::channel::<std::io::Error>(1);
let rx = rx.into_recv_async();
let span = tracing::info_span!(parent: None, "Background process task", %command);
span.follows_from(Span::current());
let handle = tracing::trace_span!(parent: None, "Spawn task", %command).in_scope(|| {
actix_rt::spawn(
async move {
let child_fut = async {
(f)(stdin).await?;
let handle = crate::sync::spawn(
async move {
let child_fut = async {
(f)(stdin).await?;
child.wait().await
};
child.wait().await
};
let error = match actix_rt::time::timeout(timeout, child_fut).await {
Ok(Ok(status)) if status.success() => {
guard.disarm();
return;
}
Ok(Ok(status)) => {
std::io::Error::new(std::io::ErrorKind::Other, StatusError(status))
}
Ok(Err(e)) => e,
Err(_) => std::io::ErrorKind::TimedOut.into(),
};
let error = match child_fut.with_timeout(timeout).await {
Ok(Ok(status)) if status.success() => {
guard.disarm();
return;
}
Ok(Ok(status)) => {
std::io::Error::new(std::io::ErrorKind::Other, StatusError(status))
}
Ok(Err(e)) => e,
Err(_) => std::io::ErrorKind::TimedOut.into(),
};
let _ = tx.send(error);
let _ = child.kill().await;
}
.instrument(span),
)
});
let _ = tx.send(error);
let _ = child.kill().await;
}
.instrument(span),
);
let sleep = actix_rt::time::sleep(timeout);

110
src/queue.rs
View File

@ -3,15 +3,13 @@ use crate::{
config::Configuration,
error::{Error, UploadError},
formats::InputProcessableFormat,
future::LocalBoxFuture,
repo::{Alias, DeleteToken, FullRepo, Hash, JobId, UploadId},
serde_str::Serde,
store::{Identifier, Store},
store::Store,
};
use base64::{prelude::BASE64_STANDARD, Engine};
use std::{
future::Future,
path::PathBuf,
pin::Pin,
sync::Arc,
time::{Duration, Instant},
};
@ -20,32 +18,6 @@ use tracing::Instrument;
mod cleanup;
mod process;
#[derive(Debug)]
struct Base64Bytes(Vec<u8>);
impl serde::Serialize for Base64Bytes {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let s = BASE64_STANDARD.encode(&self.0);
s.serialize(serializer)
}
}
impl<'de> serde::Deserialize<'de> for Base64Bytes {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let s: String = serde::Deserialize::deserialize(deserializer)?;
BASE64_STANDARD
.decode(s)
.map(Base64Bytes)
.map_err(|e| serde::de::Error::custom(e.to_string()))
}
}
const CLEANUP_QUEUE: &str = "cleanup";
const PROCESS_QUEUE: &str = "process";
@ -55,7 +27,7 @@ enum Cleanup {
hash: Hash,
},
Identifier {
identifier: Base64Bytes,
identifier: String,
},
Alias {
alias: Serde<Alias>,
@ -74,7 +46,7 @@ enum Cleanup {
#[derive(Debug, serde::Deserialize, serde::Serialize)]
enum Process {
Ingest {
identifier: Base64Bytes,
identifier: String,
upload_id: Serde<UploadId>,
declared_alias: Option<Serde<Alias>>,
},
@ -91,30 +63,30 @@ pub(crate) async fn cleanup_alias(
alias: Alias,
token: DeleteToken,
) -> Result<(), Error> {
let job = serde_json::to_vec(&Cleanup::Alias {
let job = serde_json::to_value(Cleanup::Alias {
alias: Serde::new(alias),
token: Serde::new(token),
})
.map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job.into()).await?;
repo.push(CLEANUP_QUEUE, job).await?;
Ok(())
}
pub(crate) async fn cleanup_hash(repo: &Arc<dyn FullRepo>, hash: Hash) -> Result<(), Error> {
let job = serde_json::to_vec(&Cleanup::Hash { hash }).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job.into()).await?;
let job = serde_json::to_value(Cleanup::Hash { hash }).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job).await?;
Ok(())
}
pub(crate) async fn cleanup_identifier<I: Identifier>(
pub(crate) async fn cleanup_identifier(
repo: &Arc<dyn FullRepo>,
identifier: I,
identifier: &Arc<str>,
) -> Result<(), Error> {
let job = serde_json::to_vec(&Cleanup::Identifier {
identifier: Base64Bytes(identifier.to_bytes()?),
let job = serde_json::to_value(Cleanup::Identifier {
identifier: identifier.to_string(),
})
.map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job.into()).await?;
repo.push(CLEANUP_QUEUE, job).await?;
Ok(())
}
@ -124,42 +96,42 @@ async fn cleanup_variants(
variant: Option<String>,
) -> Result<(), Error> {
let job =
serde_json::to_vec(&Cleanup::Variant { hash, variant }).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job.into()).await?;
serde_json::to_value(Cleanup::Variant { hash, variant }).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job).await?;
Ok(())
}
pub(crate) async fn cleanup_outdated_proxies(repo: &Arc<dyn FullRepo>) -> Result<(), Error> {
let job = serde_json::to_vec(&Cleanup::OutdatedProxies).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job.into()).await?;
let job = serde_json::to_value(Cleanup::OutdatedProxies).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job).await?;
Ok(())
}
pub(crate) async fn cleanup_outdated_variants(repo: &Arc<dyn FullRepo>) -> Result<(), Error> {
let job = serde_json::to_vec(&Cleanup::OutdatedVariants).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job.into()).await?;
let job = serde_json::to_value(Cleanup::OutdatedVariants).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job).await?;
Ok(())
}
pub(crate) async fn cleanup_all_variants(repo: &Arc<dyn FullRepo>) -> Result<(), Error> {
let job = serde_json::to_vec(&Cleanup::AllVariants).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job.into()).await?;
let job = serde_json::to_value(Cleanup::AllVariants).map_err(UploadError::PushJob)?;
repo.push(CLEANUP_QUEUE, job).await?;
Ok(())
}
pub(crate) async fn queue_ingest(
repo: &Arc<dyn FullRepo>,
identifier: Vec<u8>,
identifier: &Arc<str>,
upload_id: UploadId,
declared_alias: Option<Alias>,
) -> Result<(), Error> {
let job = serde_json::to_vec(&Process::Ingest {
identifier: Base64Bytes(identifier),
let job = serde_json::to_value(Process::Ingest {
identifier: identifier.to_string(),
declared_alias: declared_alias.map(Serde::new),
upload_id: Serde::new(upload_id),
})
.map_err(UploadError::PushJob)?;
repo.push(PROCESS_QUEUE, job.into()).await?;
repo.push(PROCESS_QUEUE, job).await?;
Ok(())
}
@ -170,14 +142,14 @@ pub(crate) async fn queue_generate(
process_path: PathBuf,
process_args: Vec<String>,
) -> Result<(), Error> {
let job = serde_json::to_vec(&Process::Generate {
let job = serde_json::to_value(Process::Generate {
target_format,
source: Serde::new(source),
process_path,
process_args,
})
.map_err(UploadError::PushJob)?;
repo.push(PROCESS_QUEUE, job.into()).await?;
repo.push(PROCESS_QUEUE, job).await?;
Ok(())
}
@ -206,8 +178,6 @@ pub(crate) async fn process_images<S: Store + 'static>(
.await
}
type LocalBoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + 'a>>;
async fn process_jobs<S, F>(
repo: &Arc<dyn FullRepo>,
store: &S,
@ -220,7 +190,7 @@ async fn process_jobs<S, F>(
&'a Arc<dyn FullRepo>,
&'a S,
&'a Configuration,
&'a [u8],
serde_json::Value,
) -> LocalBoxFuture<'a, Result<(), Error>>
+ Copy,
{
@ -232,6 +202,11 @@ async fn process_jobs<S, F>(
if let Err(e) = res {
tracing::warn!("Error processing jobs: {}", format!("{e}"));
tracing::warn!("{}", format!("{e:?}"));
if e.is_disconnected() {
actix_rt::time::sleep(Duration::from_secs(10)).await;
}
continue;
}
@ -284,13 +259,13 @@ where
&'a Arc<dyn FullRepo>,
&'a S,
&'a Configuration,
&'a [u8],
serde_json::Value,
) -> LocalBoxFuture<'a, Result<(), Error>>
+ Copy,
{
loop {
let fut = async {
let (job_id, bytes) = repo.pop(queue, worker_id).await?;
let (job_id, job) = repo.pop(queue, worker_id).await?;
let span = tracing::info_span!("Running Job");
@ -303,7 +278,7 @@ where
queue,
worker_id,
job_id,
(callback)(repo, store, config, bytes.as_ref()),
(callback)(repo, store, config, job),
)
})
.instrument(span)
@ -337,7 +312,7 @@ async fn process_image_jobs<S, F>(
&'a S,
&'a ProcessMap,
&'a Configuration,
&'a [u8],
serde_json::Value,
) -> LocalBoxFuture<'a, Result<(), Error>>
+ Copy,
{
@ -350,6 +325,11 @@ async fn process_image_jobs<S, F>(
if let Err(e) = res {
tracing::warn!("Error processing jobs: {}", format!("{e}"));
tracing::warn!("{}", format!("{e:?}"));
if e.is_disconnected() {
actix_rt::time::sleep(Duration::from_secs(3)).await;
}
continue;
}
@ -373,13 +353,13 @@ where
&'a S,
&'a ProcessMap,
&'a Configuration,
&'a [u8],
serde_json::Value,
) -> LocalBoxFuture<'a, Result<(), Error>>
+ Copy,
{
loop {
let fut = async {
let (job_id, bytes) = repo.pop(queue, worker_id).await?;
let (job_id, job) = repo.pop(queue, worker_id).await?;
let span = tracing::info_span!("Running Job");
@ -392,7 +372,7 @@ where
queue,
worker_id,
job_id,
(callback)(repo, store, process_map, config, bytes.as_ref()),
(callback)(repo, store, process_map, config, job),
)
})
.instrument(span)

31
src/queue/cleanup.rs
View File

@ -1,10 +1,13 @@
use std::sync::Arc;
use crate::{
config::Configuration,
error::{Error, UploadError},
queue::{Base64Bytes, Cleanup, LocalBoxFuture},
future::LocalBoxFuture,
queue::Cleanup,
repo::{Alias, ArcRepo, DeleteToken, Hash},
serde_str::Serde,
store::{Identifier, Store},
store::Store,
stream::IntoStreamer,
};
@ -12,18 +15,18 @@ pub(super) fn perform<'a, S>(
repo: &'a ArcRepo,
store: &'a S,
configuration: &'a Configuration,
job: &'a [u8],
job: serde_json::Value,
) -> LocalBoxFuture<'a, Result<(), Error>>
where
S: Store,
{
Box::pin(async move {
match serde_json::from_slice(job) {
match serde_json::from_value(job) {
Ok(job) => match job {
Cleanup::Hash { hash: in_hash } => hash(repo, in_hash).await?,
Cleanup::Identifier {
identifier: Base64Bytes(in_identifier),
} => identifier(repo, store, in_identifier).await?,
identifier: in_identifier,
} => identifier(repo, store, Arc::from(in_identifier)).await?,
Cleanup::Alias {
alias: stored_alias,
token,
@ -50,20 +53,18 @@ where
}
#[tracing::instrument(skip_all)]
async fn identifier<S>(repo: &ArcRepo, store: &S, identifier: Vec<u8>) -> Result<(), Error>
async fn identifier<S>(repo: &ArcRepo, store: &S, identifier: Arc<str>) -> Result<(), Error>
where
S: Store,
{
let identifier = S::Identifier::from_bytes(identifier)?;
let mut errors = Vec::new();
if let Err(e) = store.remove(&identifier).await {
errors.push(e);
errors.push(UploadError::from(e));
}
if let Err(e) = repo.cleanup_details(&identifier).await {
errors.push(e);
errors.push(UploadError::from(e));
}
for error in errors {
@ -100,7 +101,7 @@ async fn hash(repo: &ArcRepo, hash: Hash) -> Result<(), Error> {
idents.extend(repo.motion_identifier(hash.clone()).await?);
for identifier in idents {
let _ = super::cleanup_identifier(repo, identifier).await;
let _ = super::cleanup_identifier(repo, &identifier).await;
}
repo.cleanup_hash(hash).await?;
@ -136,7 +137,7 @@ async fn alias(repo: &ArcRepo, alias: Alias, token: DeleteToken) -> Result<(), E
#[tracing::instrument(skip_all)]
async fn all_variants(repo: &ArcRepo) -> Result<(), Error> {
let mut hash_stream = repo.hashes().await.into_streamer();
let mut hash_stream = repo.hashes().into_streamer();
while let Some(res) = hash_stream.next().await {
let hash = res?;
@ -193,7 +194,7 @@ async fn hash_variant(
.variant_identifier(hash.clone(), target_variant.clone())
.await?
{
super::cleanup_identifier(repo, identifier).await?;
super::cleanup_identifier(repo, &identifier).await?;
}
repo.remove_variant(hash.clone(), target_variant.clone())
@ -203,7 +204,7 @@ async fn hash_variant(
for (variant, identifier) in repo.variants(hash.clone()).await? {
repo.remove_variant(hash.clone(), variant.clone()).await?;
repo.remove_variant_access(hash.clone(), variant).await?;
super::cleanup_identifier(repo, identifier).await?;
super::cleanup_identifier(repo, &identifier).await?;
}
}

27
src/queue/process.rs
View File

@ -3,37 +3,38 @@ use crate::{
config::Configuration,
error::{Error, UploadError},
formats::InputProcessableFormat,
future::LocalBoxFuture,
ingest::Session,
queue::{Base64Bytes, LocalBoxFuture, Process},
queue::Process,
repo::{Alias, ArcRepo, UploadId, UploadResult},
serde_str::Serde,
store::{Identifier, Store},
store::Store,
stream::StreamMap,
};
use std::path::PathBuf;
use std::{path::PathBuf, sync::Arc};
pub(super) fn perform<'a, S>(
repo: &'a ArcRepo,
store: &'a S,
process_map: &'a ProcessMap,
config: &'a Configuration,
job: &'a [u8],
job: serde_json::Value,
) -> LocalBoxFuture<'a, Result<(), Error>>
where
S: Store + 'static,
{
Box::pin(async move {
match serde_json::from_slice(job) {
match serde_json::from_value(job) {
Ok(job) => match job {
Process::Ingest {
identifier: Base64Bytes(identifier),
identifier,
upload_id,
declared_alias,
} => {
process_ingest(
repo,
store,
identifier,
Arc::from(identifier),
Serde::into_inner(upload_id),
declared_alias.map(Serde::into_inner),
&config.media,
@ -68,11 +69,11 @@ where
})
}
#[tracing::instrument(skip_all)]
#[tracing::instrument(skip(repo, store, media))]
async fn process_ingest<S>(
repo: &ArcRepo,
store: &S,
unprocessed_identifier: Vec<u8>,
unprocessed_identifier: Arc<str>,
upload_id: UploadId,
declared_alias: Option<Alias>,
media: &crate::config::Media,
@ -81,14 +82,12 @@ where
S: Store + 'static,
{
let fut = async {
let unprocessed_identifier = S::Identifier::from_bytes(unprocessed_identifier)?;
let ident = unprocessed_identifier.clone();
let store2 = store.clone();
let repo = repo.clone();
let media = media.clone();
let error_boundary = actix_rt::spawn(async move {
let error_boundary = crate::sync::spawn(async move {
let stream = store2
.to_stream(&ident, None, None)
.await?
@ -97,7 +96,7 @@ where
let session =
crate::ingest::ingest(&repo, &store2, stream, declared_alias, &media).await?;
Ok(session) as Result<Session<S>, Error>
Ok(session) as Result<Session, Error>
})
.await;
@ -128,7 +127,7 @@ where
}
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(skip_all)]
#[tracing::instrument(skip(repo, store, process_map, process_path, process_args, config))]
async fn generate<S: Store + 'static>(
repo: &ArcRepo,
store: &S,

4
src/range.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use crate::{
error::{Error, UploadError},
store::Store,
@ -26,7 +28,7 @@ pub(crate) fn chop_bytes(
pub(crate) async fn chop_store<S: Store>(
byte_range: &ByteRangeSpec,
store: &S,
identifier: &S::Identifier,
identifier: &Arc<str>,
length: u64,
) -> Result<impl Stream<Item = std::io::Result<Bytes>>, Error> {
if let Some((start, end)) = byte_range.to_satisfiable_range(length) {

580
src/repo.rs
View File

@ -1,8 +1,14 @@
mod alias;
mod delete_token;
mod hash;
mod metrics;
mod migrate;
use crate::{
config,
details::Details,
error_code::{ErrorCode, OwnedErrorCode},
store::{Identifier, StoreError},
future::LocalBoxFuture,
stream::LocalBoxStream,
};
use base64::Engine;
@ -10,10 +16,11 @@ use std::{fmt::Debug, sync::Arc};
use url::Url;
use uuid::Uuid;
mod hash;
mod migrate;
pub(crate) mod postgres;
pub(crate) mod sled;
pub(crate) use alias::Alias;
pub(crate) use delete_token::DeleteToken;
pub(crate) use hash::Hash;
pub(crate) use migrate::{migrate_04, migrate_repo};
@ -22,6 +29,7 @@ pub(crate) type ArcRepo = Arc<dyn FullRepo>;
#[derive(Clone, Debug)]
pub(crate) enum Repo {
Sled(self::sled::SledRepo),
Postgres(self::postgres::PostgresRepo),
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
@ -30,17 +38,6 @@ enum MaybeUuid {
Name(String),
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub(crate) struct Alias {
id: MaybeUuid,
extension: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub(crate) struct DeleteToken {
id: MaybeUuid,
}
#[derive(Debug)]
pub(crate) struct HashAlreadyExists;
#[derive(Debug)]
@ -70,6 +67,9 @@ pub(crate) enum RepoError {
#[error("Error in sled")]
SledError(#[from] crate::repo::sled::SledError),
#[error("Error in postgres")]
PostgresError(#[from] crate::repo::postgres::PostgresError),
#[error("Upload was already claimed")]
AlreadyClaimed,
@ -81,10 +81,18 @@ impl RepoError {
pub(crate) const fn error_code(&self) -> ErrorCode {
match self {
Self::SledError(e) => e.error_code(),
Self::PostgresError(e) => e.error_code(),
Self::AlreadyClaimed => ErrorCode::ALREADY_CLAIMED,
Self::Canceled => ErrorCode::PANIC,
}
}
pub(crate) const fn is_disconnected(&self) -> bool {
match self {
Self::PostgresError(e) => e.is_disconnected(),
_ => false,
}
}
}
#[async_trait::async_trait(?Send)]
@ -106,7 +114,7 @@ pub(crate) trait FullRepo:
async fn health_check(&self) -> Result<(), RepoError>;
#[tracing::instrument(skip(self))]
async fn identifier_from_alias(&self, alias: &Alias) -> Result<Option<Arc<[u8]>>, RepoError> {
async fn identifier_from_alias(&self, alias: &Alias) -> Result<Option<Arc<str>>, RepoError> {
let Some(hash) = self.hash(alias).await? else {
return Ok(None);
};
@ -127,7 +135,7 @@ pub(crate) trait FullRepo:
async fn still_identifier_from_alias(
&self,
alias: &Alias,
) -> Result<Option<Arc<[u8]>>, StoreError> {
) -> Result<Option<Arc<str>>, RepoError> {
let Some(hash) = self.hash(alias).await? else {
return Ok(None);
};
@ -367,13 +375,13 @@ impl JobId {
#[async_trait::async_trait(?Send)]
pub(crate) trait QueueRepo: BaseRepo {
async fn push(&self, queue: &'static str, job: Arc<[u8]>) -> Result<JobId, RepoError>;
async fn push(&self, queue: &'static str, job: serde_json::Value) -> Result<JobId, RepoError>;
async fn pop(
&self,
queue: &'static str,
worker_id: Uuid,
) -> Result<(JobId, Arc<[u8]>), RepoError>;
) -> Result<(JobId, serde_json::Value), RepoError>;
async fn heartbeat(
&self,
@ -395,7 +403,7 @@ impl<T> QueueRepo for Arc<T>
where
T: QueueRepo,
{
async fn push(&self, queue: &'static str, job: Arc<[u8]>) -> Result<JobId, RepoError> {
async fn push(&self, queue: &'static str, job: serde_json::Value) -> Result<JobId, RepoError> {
T::push(self, queue, job).await
}
@ -403,7 +411,7 @@ where
&self,
queue: &'static str,
worker_id: Uuid,
) -> Result<(JobId, Arc<[u8]>), RepoError> {
) -> Result<(JobId, serde_json::Value), RepoError> {
T::pop(self, queue, worker_id).await
}
@ -455,12 +463,12 @@ where
pub(crate) trait DetailsRepo: BaseRepo {
async fn relate_details(
&self,
identifier: &dyn Identifier,
identifier: &Arc<str>,
details: &Details,
) -> Result<(), StoreError>;
async fn details(&self, identifier: &dyn Identifier) -> Result<Option<Details>, StoreError>;
) -> Result<(), RepoError>;
async fn details(&self, identifier: &Arc<str>) -> Result<Option<Details>, RepoError>;
async fn cleanup_details(&self, identifier: &dyn Identifier) -> Result<(), StoreError>;
async fn cleanup_details(&self, identifier: &Arc<str>) -> Result<(), RepoError>;
}
#[async_trait::async_trait(?Send)]
@ -470,17 +478,17 @@ where
{
async fn relate_details(
&self,
identifier: &dyn Identifier,
identifier: &Arc<str>,
details: &Details,
) -> Result<(), StoreError> {
) -> Result<(), RepoError> {
T::relate_details(self, identifier, details).await
}
async fn details(&self, identifier: &dyn Identifier) -> Result<Option<Details>, StoreError> {
async fn details(&self, identifier: &Arc<str>) -> Result<Option<Details>, RepoError> {
T::details(self, identifier).await
}
async fn cleanup_details(&self, identifier: &dyn Identifier) -> Result<(), StoreError> {
async fn cleanup_details(&self, identifier: &Arc<str>) -> Result<(), RepoError> {
T::cleanup_details(self, identifier).await
}
}
@ -491,11 +499,11 @@ pub(crate) trait StoreMigrationRepo: BaseRepo {
async fn mark_migrated(
&self,
old_identifier: &dyn Identifier,
new_identifier: &dyn Identifier,
) -> Result<(), StoreError>;
old_identifier: &Arc<str>,
new_identifier: &Arc<str>,
) -> Result<(), RepoError>;
async fn is_migrated(&self, identifier: &dyn Identifier) -> Result<bool, StoreError>;
async fn is_migrated(&self, identifier: &Arc<str>) -> Result<bool, RepoError>;
async fn clear(&self) -> Result<(), RepoError>;
}
@ -511,13 +519,13 @@ where
async fn mark_migrated(
&self,
old_identifier: &dyn Identifier,
new_identifier: &dyn Identifier,
) -> Result<(), StoreError> {
old_identifier: &Arc<str>,
new_identifier: &Arc<str>,
) -> Result<(), RepoError> {
T::mark_migrated(self, old_identifier, new_identifier).await
}
async fn is_migrated(&self, identifier: &dyn Identifier) -> Result<bool, StoreError> {
async fn is_migrated(&self, identifier: &Arc<str>) -> Result<bool, RepoError> {
T::is_migrated(self, identifier).await
}
@ -526,7 +534,7 @@ where
}
}
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) struct OrderedHash {
timestamp: time::OffsetDateTime,
hash: Hash,
@ -564,12 +572,88 @@ impl HashPage {
}
}
type PageFuture = LocalBoxFuture<'static, Result<HashPage, RepoError>>;
pub(crate) struct HashStream {
repo: Option<ArcRepo>,
page_future: Option<PageFuture>,
page: Option<HashPage>,
}
impl futures_core::Stream for HashStream {
type Item = Result<Hash, RepoError>;
fn poll_next(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
let this = self.get_mut();
loop {
let Some(repo) = &this.repo else {
return std::task::Poll::Ready(None);
};
let slug = if let Some(page) = &mut this.page {
// popping last in page is fine - we reversed them
if let Some(hash) = page.hashes.pop() {
return std::task::Poll::Ready(Some(Ok(hash)));
}
let slug = page.next();
this.page.take();
if let Some(slug) = slug {
Some(slug)
} else {
this.repo.take();
return std::task::Poll::Ready(None);
}
} else {
None
};
if let Some(page_future) = &mut this.page_future {
let res = std::task::ready!(page_future.as_mut().poll(cx));
this.page_future.take();
match res {
Ok(mut page) => {
// reverse because we use `pop` to fetch next
page.hashes.reverse();
this.page = Some(page);
}
Err(e) => {
this.repo.take();
return std::task::Poll::Ready(Some(Err(e)));
}
}
} else {
let repo = repo.clone();
this.page_future = Some(Box::pin(async move { repo.hash_page(slug, 100).await }));
}
}
}
}
impl dyn FullRepo {
pub(crate) fn hashes(self: &Arc<Self>) -> HashStream {
HashStream {
repo: Some(self.clone()),
page_future: None,
page: None,
}
}
}
#[async_trait::async_trait(?Send)]
pub(crate) trait HashRepo: BaseRepo {
async fn size(&self) -> Result<u64, RepoError>;
async fn hashes(&self) -> LocalBoxStream<'static, Result<Hash, RepoError>>;
async fn hash_page(&self, slug: Option<String>, limit: usize) -> Result<HashPage, RepoError> {
let hash = slug.as_deref().and_then(hash_from_slug);
@ -599,8 +683,8 @@ pub(crate) trait HashRepo: BaseRepo {
async fn create_hash(
&self,
hash: Hash,
identifier: &dyn Identifier,
) -> Result<Result<(), HashAlreadyExists>, StoreError> {
identifier: &Arc<str>,
) -> Result<Result<(), HashAlreadyExists>, RepoError> {
self.create_hash_with_timestamp(hash, identifier, time::OffsetDateTime::now_utc())
.await
}
@ -608,38 +692,34 @@ pub(crate) trait HashRepo: BaseRepo {
async fn create_hash_with_timestamp(
&self,
hash: Hash,
identifier: &dyn Identifier,
identifier: &Arc<str>,
timestamp: time::OffsetDateTime,
) -> Result<Result<(), HashAlreadyExists>, StoreError>;
) -> Result<Result<(), HashAlreadyExists>, RepoError>;
async fn update_identifier(
&self,
hash: Hash,
identifier: &dyn Identifier,
) -> Result<(), StoreError>;
async fn update_identifier(&self, hash: Hash, identifier: &Arc<str>) -> Result<(), RepoError>;
async fn identifier(&self, hash: Hash) -> Result<Option<Arc<[u8]>>, RepoError>;
async fn identifier(&self, hash: Hash) -> Result<Option<Arc<str>>, RepoError>;
async fn relate_variant_identifier(
&self,
hash: Hash,
variant: String,
identifier: &dyn Identifier,
) -> Result<Result<(), VariantAlreadyExists>, StoreError>;
identifier: &Arc<str>,
) -> Result<Result<(), VariantAlreadyExists>, RepoError>;
async fn variant_identifier(
&self,
hash: Hash,
variant: String,
) -> Result<Option<Arc<[u8]>>, RepoError>;
async fn variants(&self, hash: Hash) -> Result<Vec<(String, Arc<[u8]>)>, RepoError>;
) -> Result<Option<Arc<str>>, RepoError>;
async fn variants(&self, hash: Hash) -> Result<Vec<(String, Arc<str>)>, RepoError>;
async fn remove_variant(&self, hash: Hash, variant: String) -> Result<(), RepoError>;
async fn relate_motion_identifier(
&self,
hash: Hash,
identifier: &dyn Identifier,
) -> Result<(), StoreError>;
async fn motion_identifier(&self, hash: Hash) -> Result<Option<Arc<[u8]>>, RepoError>;
identifier: &Arc<str>,
) -> Result<(), RepoError>;
async fn motion_identifier(&self, hash: Hash) -> Result<Option<Arc<str>>, RepoError>;
async fn cleanup_hash(&self, hash: Hash) -> Result<(), RepoError>;
}
@ -653,10 +733,6 @@ where
T::size(self).await
}
async fn hashes(&self) -> LocalBoxStream<'static, Result<Hash, RepoError>> {
T::hashes(self).await
}
async fn bound(&self, hash: Hash) -> Result<Option<OrderedHash>, RepoError> {
T::bound(self, hash).await
}
@ -680,21 +756,17 @@ where
async fn create_hash_with_timestamp(
&self,
hash: Hash,
identifier: &dyn Identifier,
identifier: &Arc<str>,
timestamp: time::OffsetDateTime,
) -> Result<Result<(), HashAlreadyExists>, StoreError> {
) -> Result<Result<(), HashAlreadyExists>, RepoError> {
T::create_hash_with_timestamp(self, hash, identifier, timestamp).await
}
async fn update_identifier(
&self,
hash: Hash,
identifier: &dyn Identifier,
) -> Result<(), StoreError> {
async fn update_identifier(&self, hash: Hash, identifier: &Arc<str>) -> Result<(), RepoError> {
T::update_identifier(self, hash, identifier).await
}
async fn identifier(&self, hash: Hash) -> Result<Option<Arc<[u8]>>, RepoError> {
async fn identifier(&self, hash: Hash) -> Result<Option<Arc<str>>, RepoError> {
T::identifier(self, hash).await
}
@ -702,8 +774,8 @@ where
&self,
hash: Hash,
variant: String,
identifier: &dyn Identifier,
) -> Result<Result<(), VariantAlreadyExists>, StoreError> {
identifier: &Arc<str>,
) -> Result<Result<(), VariantAlreadyExists>, RepoError> {
T::relate_variant_identifier(self, hash, variant, identifier).await
}
@ -711,11 +783,11 @@ where
&self,
hash: Hash,
variant: String,
) -> Result<Option<Arc<[u8]>>, RepoError> {
) -> Result<Option<Arc<str>>, RepoError> {
T::variant_identifier(self, hash, variant).await
}
async fn variants(&self, hash: Hash) -> Result<Vec<(String, Arc<[u8]>)>, RepoError> {
async fn variants(&self, hash: Hash) -> Result<Vec<(String, Arc<str>)>, RepoError> {
T::variants(self, hash).await
}
@ -726,12 +798,12 @@ where
async fn relate_motion_identifier(
&self,
hash: Hash,
identifier: &dyn Identifier,
) -> Result<(), StoreError> {
identifier: &Arc<str>,
) -> Result<(), RepoError> {
T::relate_motion_identifier(self, hash, identifier).await
}
async fn motion_identifier(&self, hash: Hash) -> Result<Option<Arc<[u8]>>, RepoError> {
async fn motion_identifier(&self, hash: Hash) -> Result<Option<Arc<str>>, RepoError> {
T::motion_identifier(self, hash).await
}
@ -791,7 +863,7 @@ where
impl Repo {
#[tracing::instrument]
pub(crate) fn open(config: config::Repo) -> color_eyre::Result<Self> {
pub(crate) async fn open(config: config::Repo) -> color_eyre::Result<Self> {
match config {
config::Repo::Sled(config::Sled {
path,
@ -802,12 +874,18 @@ impl Repo {
Ok(Self::Sled(repo))
}
config::Repo::Postgres(config::Postgres { url }) => {
let repo = self::postgres::PostgresRepo::connect(url).await?;
Ok(Self::Postgres(repo))
}
}
}
pub(crate) fn to_arc(&self) -> ArcRepo {
match self {
Self::Sled(sled_repo) => Arc::new(sled_repo.clone()),
Self::Postgres(postgres_repo) => Arc::new(postgres_repo.clone()),
}
}
}
@ -829,106 +907,6 @@ impl MaybeUuid {
}
}
fn split_at_dot(s: &str) -> Option<(&str, &str)> {
let index = s.find('.')?;
Some(s.split_at(index))
}
impl Alias {
pub(crate) fn generate(extension: String) -> Self {
Alias {
id: MaybeUuid::Uuid(Uuid::new_v4()),
extension: Some(extension),
}
}
pub(crate) fn from_existing(alias: &str) -> Self {
if let Some((start, end)) = split_at_dot(alias) {
Alias {
id: MaybeUuid::from_str(start),
extension: Some(end.into()),
}
} else {
Alias {
id: MaybeUuid::from_str(alias),
extension: None,
}
}
}
pub(crate) fn extension(&self) -> Option<&str> {
self.extension.as_deref()
}
pub(crate) fn to_bytes(&self) -> Vec<u8> {
let mut v = self.id.as_bytes().to_vec();
if let Some(ext) = self.extension() {
v.extend_from_slice(ext.as_bytes());
}
v
}
pub(crate) fn from_slice(bytes: &[u8]) -> Option<Self> {
if let Ok(s) = std::str::from_utf8(bytes) {
Some(Self::from_existing(s))
} else if bytes.len() >= 16 {
let id = Uuid::from_slice(&bytes[0..16]).expect("Already checked length");
let extension = if bytes.len() > 16 {
Some(String::from_utf8_lossy(&bytes[16..]).to_string())
} else {
None
};
Some(Self {
id: MaybeUuid::Uuid(id),
extension,
})
} else {
None
}
}
}
impl DeleteToken {
pub(crate) fn from_existing(existing: &str) -> Self {
if let Ok(uuid) = Uuid::parse_str(existing) {
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
} else {
DeleteToken {
id: MaybeUuid::Name(existing.into()),
}
}
}
pub(crate) fn generate() -> Self {
Self {
id: MaybeUuid::Uuid(Uuid::new_v4()),
}
}
pub(crate) fn to_bytes(&self) -> Vec<u8> {
self.id.as_bytes().to_vec()
}
pub(crate) fn from_slice(bytes: &[u8]) -> Option<Self> {
if let Ok(s) = std::str::from_utf8(bytes) {
Some(DeleteToken::from_existing(s))
} else if bytes.len() == 16 {
Some(DeleteToken {
id: MaybeUuid::Uuid(Uuid::from_slice(bytes).ok()?),
})
} else {
None
}
}
}
impl UploadId {
pub(crate) fn generate() -> Self {
Self { id: Uuid::new_v4() }
@ -961,253 +939,3 @@ impl std::fmt::Display for MaybeUuid {
}
}
}
impl std::str::FromStr for DeleteToken {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(DeleteToken::from_existing(s))
}
}
impl std::fmt::Display for DeleteToken {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.id)
}
}
impl std::str::FromStr for Alias {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Alias::from_existing(s))
}
}
impl std::fmt::Display for Alias {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if let Some(ext) = self.extension() {
write!(f, "{}{ext}", self.id)
} else {
write!(f, "{}", self.id)
}
}
}
#[cfg(test)]
mod tests {
use super::{Alias, DeleteToken, MaybeUuid, Uuid};
#[test]
fn string_delete_token() {
let delete_token = DeleteToken::from_existing("blah");
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Name(String::from("blah"))
}
)
}
#[test]
fn uuid_string_delete_token() {
let uuid = Uuid::new_v4();
let delete_token = DeleteToken::from_existing(&uuid.to_string());
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
)
}
#[test]
fn bytes_delete_token() {
let delete_token = DeleteToken::from_slice(b"blah").unwrap();
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Name(String::from("blah"))
}
)
}
#[test]
fn uuid_bytes_delete_token() {
let uuid = Uuid::new_v4();
let delete_token = DeleteToken::from_slice(&uuid.as_bytes()[..]).unwrap();
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
)
}
#[test]
fn uuid_bytes_string_delete_token() {
let uuid = Uuid::new_v4();
let delete_token = DeleteToken::from_slice(uuid.to_string().as_bytes()).unwrap();
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
)
}
#[test]
fn string_alias() {
let alias = Alias::from_existing("blah");
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: None
}
);
}
#[test]
fn string_alias_ext() {
let alias = Alias::from_existing("blah.mp4");
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: Some(String::from(".mp4")),
}
);
}
#[test]
fn uuid_string_alias() {
let uuid = Uuid::new_v4();
let alias = Alias::from_existing(&uuid.to_string());
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: None,
}
)
}
#[test]
fn uuid_string_alias_ext() {
let uuid = Uuid::new_v4();
let alias_str = format!("{uuid}.mp4");
let alias = Alias::from_existing(&alias_str);
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: Some(String::from(".mp4")),
}
)
}
#[test]
fn bytes_alias() {
let alias = Alias::from_slice(b"blah").unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: None
}
);
}
#[test]
fn bytes_alias_ext() {
let alias = Alias::from_slice(b"blah.mp4").unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: Some(String::from(".mp4")),
}
);
}
#[test]
fn uuid_bytes_alias() {
let uuid = Uuid::new_v4();
let alias = Alias::from_slice(&uuid.as_bytes()[..]).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: None,
}
)
}
#[test]
fn uuid_bytes_string_alias() {
let uuid = Uuid::new_v4();
let alias = Alias::from_slice(uuid.to_string().as_bytes()).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: None,
}
)
}
#[test]
fn uuid_bytes_alias_ext() {
let uuid = Uuid::new_v4();
let mut alias_bytes = uuid.as_bytes().to_vec();
alias_bytes.extend_from_slice(b".mp4");
let alias = Alias::from_slice(&alias_bytes).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: Some(String::from(".mp4")),
}
)
}
#[test]
fn uuid_bytes_string_alias_ext() {
let uuid = Uuid::new_v4();
let alias_str = format!("{uuid}.mp4");
let alias = Alias::from_slice(alias_str.as_bytes()).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: Some(String::from(".mp4")),
}
)
}
}

274
src/repo/alias.rs Normal file
View File

@ -0,0 +1,274 @@
use diesel::{backend::Backend, sql_types::VarChar, AsExpression, FromSqlRow};
use uuid::Uuid;
use super::MaybeUuid;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, AsExpression, FromSqlRow)]
#[diesel(sql_type = VarChar)]
pub(crate) struct Alias {
id: MaybeUuid,
extension: Option<String>,
}
impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for Alias {
fn to_sql<'b>(
&'b self,
out: &mut diesel::serialize::Output<'b, '_, diesel::pg::Pg>,
) -> diesel::serialize::Result {
let s = self.to_string();
<String as diesel::serialize::ToSql<VarChar, diesel::pg::Pg>>::to_sql(
&s,
&mut out.reborrow(),
)
}
}
impl<B> diesel::deserialize::FromSql<VarChar, B> for Alias
where
B: Backend,
String: diesel::deserialize::FromSql<VarChar, B>,
{
fn from_sql(
bytes: <B as diesel::backend::Backend>::RawValue<'_>,
) -> diesel::deserialize::Result<Self> {
let s = String::from_sql(bytes)?;
s.parse().map_err(From::from)
}
}
impl Alias {
pub(crate) fn generate(extension: String) -> Self {
Alias {
id: MaybeUuid::Uuid(Uuid::new_v4()),
extension: Some(extension),
}
}
pub(crate) fn from_existing(alias: &str) -> Self {
if let Some((start, end)) = split_at_dot(alias) {
Alias {
id: MaybeUuid::from_str(start),
extension: Some(end.into()),
}
} else {
Alias {
id: MaybeUuid::from_str(alias),
extension: None,
}
}
}
pub(crate) fn extension(&self) -> Option<&str> {
self.extension.as_deref()
}
pub(crate) fn to_bytes(&self) -> Vec<u8> {
let mut v = self.id.as_bytes().to_vec();
if let Some(ext) = self.extension() {
v.extend_from_slice(ext.as_bytes());
}
v
}
pub(crate) fn from_slice(bytes: &[u8]) -> Option<Self> {
if let Ok(s) = std::str::from_utf8(bytes) {
Some(Self::from_existing(s))
} else if bytes.len() >= 16 {
let id = Uuid::from_slice(&bytes[0..16]).expect("Already checked length");
let extension = if bytes.len() > 16 {
Some(String::from_utf8_lossy(&bytes[16..]).to_string())
} else {
None
};
Some(Self {
id: MaybeUuid::Uuid(id),
extension,
})
} else {
None
}
}
}
fn split_at_dot(s: &str) -> Option<(&str, &str)> {
let index = s.find('.')?;
Some(s.split_at(index))
}
impl std::str::FromStr for Alias {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Alias::from_existing(s))
}
}
impl std::fmt::Display for Alias {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if let Some(ext) = self.extension() {
write!(f, "{}{ext}", self.id)
} else {
write!(f, "{}", self.id)
}
}
}
#[cfg(test)]
mod tests {
use super::{Alias, MaybeUuid};
use uuid::Uuid;
#[test]
fn string_alias() {
let alias = Alias::from_existing("blah");
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: None
}
);
}
#[test]
fn string_alias_ext() {
let alias = Alias::from_existing("blah.mp4");
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: Some(String::from(".mp4")),
}
);
}
#[test]
fn uuid_string_alias() {
let uuid = Uuid::new_v4();
let alias = Alias::from_existing(&uuid.to_string());
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: None,
}
)
}
#[test]
fn uuid_string_alias_ext() {
let uuid = Uuid::new_v4();
let alias_str = format!("{uuid}.mp4");
let alias = Alias::from_existing(&alias_str);
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: Some(String::from(".mp4")),
}
)
}
#[test]
fn bytes_alias() {
let alias = Alias::from_slice(b"blah").unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: None
}
);
}
#[test]
fn bytes_alias_ext() {
let alias = Alias::from_slice(b"blah.mp4").unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Name(String::from("blah")),
extension: Some(String::from(".mp4")),
}
);
}
#[test]
fn uuid_bytes_alias() {
let uuid = Uuid::new_v4();
let alias = Alias::from_slice(&uuid.as_bytes()[..]).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: None,
}
)
}
#[test]
fn uuid_bytes_string_alias() {
let uuid = Uuid::new_v4();
let alias = Alias::from_slice(uuid.to_string().as_bytes()).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: None,
}
)
}
#[test]
fn uuid_bytes_alias_ext() {
let uuid = Uuid::new_v4();
let mut alias_bytes = uuid.as_bytes().to_vec();
alias_bytes.extend_from_slice(b".mp4");
let alias = Alias::from_slice(&alias_bytes).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: Some(String::from(".mp4")),
}
)
}
#[test]
fn uuid_bytes_string_alias_ext() {
let uuid = Uuid::new_v4();
let alias_str = format!("{uuid}.mp4");
let alias = Alias::from_slice(alias_str.as_bytes()).unwrap();
assert_eq!(
alias,
Alias {
id: MaybeUuid::Uuid(uuid),
extension: Some(String::from(".mp4")),
}
)
}
}

160
src/repo/delete_token.rs Normal file
View File

@ -0,0 +1,160 @@
use diesel::{backend::Backend, sql_types::VarChar, AsExpression, FromSqlRow};
use uuid::Uuid;
use super::MaybeUuid;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, AsExpression, FromSqlRow)]
#[diesel(sql_type = VarChar)]
pub(crate) struct DeleteToken {
id: MaybeUuid,
}
impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for DeleteToken {
fn to_sql<'b>(
&'b self,
out: &mut diesel::serialize::Output<'b, '_, diesel::pg::Pg>,
) -> diesel::serialize::Result {
let s = self.to_string();
<String as diesel::serialize::ToSql<VarChar, diesel::pg::Pg>>::to_sql(
&s,
&mut out.reborrow(),
)
}
}
impl<B> diesel::deserialize::FromSql<VarChar, B> for DeleteToken
where
B: Backend,
String: diesel::deserialize::FromSql<VarChar, B>,
{
fn from_sql(
bytes: <B as diesel::backend::Backend>::RawValue<'_>,
) -> diesel::deserialize::Result<Self> {
let s = String::from_sql(bytes)?;
s.parse().map_err(From::from)
}
}
impl DeleteToken {
pub(crate) fn from_existing(existing: &str) -> Self {
if let Ok(uuid) = Uuid::parse_str(existing) {
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
} else {
DeleteToken {
id: MaybeUuid::Name(existing.into()),
}
}
}
pub(crate) fn generate() -> Self {
Self {
id: MaybeUuid::Uuid(Uuid::new_v4()),
}
}
pub(crate) fn to_bytes(&self) -> Vec<u8> {
self.id.as_bytes().to_vec()
}
pub(crate) fn from_slice(bytes: &[u8]) -> Option<Self> {
if let Ok(s) = std::str::from_utf8(bytes) {
Some(DeleteToken::from_existing(s))
} else if bytes.len() == 16 {
Some(DeleteToken {
id: MaybeUuid::Uuid(Uuid::from_slice(bytes).ok()?),
})
} else {
None
}
}
}
impl std::str::FromStr for DeleteToken {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(DeleteToken::from_existing(s))
}
}
impl std::fmt::Display for DeleteToken {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.id)
}
}
#[cfg(test)]
mod tests {
use super::{DeleteToken, MaybeUuid};
use uuid::Uuid;
#[test]
fn string_delete_token() {
let delete_token = DeleteToken::from_existing("blah");
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Name(String::from("blah"))
}
)
}
#[test]
fn uuid_string_delete_token() {
let uuid = Uuid::new_v4();
let delete_token = DeleteToken::from_existing(&uuid.to_string());
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
)
}
#[test]
fn bytes_delete_token() {
let delete_token = DeleteToken::from_slice(b"blah").unwrap();
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Name(String::from("blah"))
}
)
}
#[test]
fn uuid_bytes_delete_token() {
let uuid = Uuid::new_v4();
let delete_token = DeleteToken::from_slice(&uuid.as_bytes()[..]).unwrap();
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
)
}
#[test]
fn uuid_bytes_string_delete_token() {
let uuid = Uuid::new_v4();
let delete_token = DeleteToken::from_slice(uuid.to_string().as_bytes()).unwrap();
assert_eq!(
delete_token,
DeleteToken {
id: MaybeUuid::Uuid(uuid),
}
)
}
}

47
src/repo/hash.rs
View File

@ -1,13 +1,42 @@
use diesel::{backend::Backend, sql_types::VarChar, AsExpression, FromSqlRow};
use crate::formats::InternalFormat;
use std::sync::Arc;
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, AsExpression, FromSqlRow)]
#[diesel(sql_type = VarChar)]
pub(crate) struct Hash {
hash: Arc<[u8; 32]>,
size: u64,
format: InternalFormat,
}
impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for Hash {
fn to_sql<'b>(
&'b self,
out: &mut diesel::serialize::Output<'b, '_, diesel::pg::Pg>,
) -> diesel::serialize::Result {
let s = self.to_base64();
<String as diesel::serialize::ToSql<VarChar, diesel::pg::Pg>>::to_sql(
&s,
&mut out.reborrow(),
)
}
}
impl<B> diesel::deserialize::FromSql<VarChar, B> for Hash
where
B: Backend,
String: diesel::deserialize::FromSql<VarChar, B>,
{
fn from_sql(bytes: <B as Backend>::RawValue<'_>) -> diesel::deserialize::Result<Self> {
let s = String::from_sql(bytes)?;
Self::from_base64(s).ok_or_else(|| "Invalid base64 hash".to_string().into())
}
}
impl Hash {
pub(crate) fn new(hash: [u8; 32], size: u64, format: InternalFormat) -> Self {
Self {
@ -30,6 +59,22 @@ impl Hash {
hex::encode(self.to_bytes())
}
pub(crate) fn to_base64(&self) -> String {
use base64::Engine;
base64::engine::general_purpose::STANDARD.encode(self.to_bytes())
}
pub(crate) fn from_base64(input: String) -> Option<Self> {
use base64::Engine;
let bytes = base64::engine::general_purpose::STANDARD
.decode(input)
.ok()?;
Self::from_bytes(&bytes)
}
pub(super) fn to_bytes(&self) -> Vec<u8> {
let format_byte = self.format.to_byte();

74
src/repo/metrics.rs Normal file
View File

@ -0,0 +1,74 @@
use std::time::Instant;
pub(super) struct PushMetricsGuard {
queue: &'static str,
armed: bool,
}
pub(super) struct PopMetricsGuard {
queue: &'static str,
start: Instant,
armed: bool,
}
pub(super) struct WaitMetricsGuard {
start: Instant,
armed: bool,
}
impl PushMetricsGuard {
pub(super) fn guard(queue: &'static str) -> Self {
Self { queue, armed: true }
}
pub(super) fn disarm(mut self) {
self.armed = false;
}
}
impl PopMetricsGuard {
pub(super) fn guard(queue: &'static str) -> Self {
Self {
queue,
start: Instant::now(),
armed: true,
}
}
pub(super) fn disarm(mut self) {
self.armed = false;
}
}
impl WaitMetricsGuard {
pub(super) fn guard() -> Self {
Self {
start: Instant::now(),
armed: true,
}
}
pub(super) fn disarm(mut self) {
self.armed = false;
}
}
impl Drop for PushMetricsGuard {
fn drop(&mut self) {
metrics::increment_counter!("pict-rs.queue.push", "completed" => (!self.armed).to_string(), "queue" => self.queue);
}
}
impl Drop for PopMetricsGuard {
fn drop(&mut self) {
metrics::histogram!("pict-rs.queue.pop.duration", self.start.elapsed().as_secs_f64(), "completed" => (!self.armed).to_string(), "queue" => self.queue);
metrics::increment_counter!("pict-rs.queue.pop", "completed" => (!self.armed).to_string(), "queue" => self.queue);
}
}
impl Drop for WaitMetricsGuard {
fn drop(&mut self) {
metrics::histogram!("pict-rs.upload.wait.duration", self.start.elapsed().as_secs_f64(), "completed" => (!self.armed).to_string());
metrics::increment_counter!("pict-rs.upload.wait", "completed" => (!self.armed).to_string());
}
}

26
src/repo/migrate.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use tokio::task::JoinSet;
use crate::{
@ -33,7 +35,7 @@ pub(crate) async fn migrate_repo(old_repo: ArcRepo, new_repo: ArcRepo) -> Result
tracing::warn!("Checks complete, migrating repo");
tracing::warn!("{total_size} hashes will be migrated");
let mut hash_stream = old_repo.hashes().await.into_streamer();
let mut hash_stream = old_repo.hashes().into_streamer();
let mut index = 0;
while let Some(res) = hash_stream.next().await {
@ -204,10 +206,14 @@ async fn do_migrate_hash(old_repo: &ArcRepo, new_repo: &ArcRepo, hash: Hash) ->
return Ok(());
};
let _ = new_repo.create_hash(hash.clone(), &identifier).await?;
if let Some(details) = old_repo.details(&identifier).await? {
let _ = new_repo
.create_hash_with_timestamp(hash.clone(), &identifier, details.created_at())
.await?;
new_repo.relate_details(&identifier, &details).await?;
} else {
let _ = new_repo.create_hash(hash.clone(), &identifier).await?;
}
if let Some(identifier) = old_repo.motion_identifier(hash.clone()).await? {
@ -266,7 +272,7 @@ async fn do_migrate_hash_04<S: Store>(
config: &Configuration,
old_hash: sled::IVec,
) -> Result<(), Error> {
let Some(identifier) = old_repo.identifier::<S::Identifier>(old_hash.clone()).await? else {
let Some(identifier) = old_repo.identifier(old_hash.clone()).await? else {
tracing::warn!("Skipping hash {}, no identifier", hex::encode(&old_hash));
return Ok(());
};
@ -276,10 +282,8 @@ async fn do_migrate_hash_04<S: Store>(
let hash_details = set_details(old_repo, new_repo, store, config, &identifier).await?;
let aliases = old_repo.aliases_for_hash(old_hash.clone()).await?;
let variants = old_repo.variants::<S::Identifier>(old_hash.clone()).await?;
let motion_identifier = old_repo
.motion_identifier::<S::Identifier>(old_hash.clone())
.await?;
let variants = old_repo.variants(old_hash.clone()).await?;
let motion_identifier = old_repo.motion_identifier(old_hash.clone()).await?;
let hash = old_hash[..].try_into().expect("Invalid hash size");
@ -326,7 +330,7 @@ async fn set_details<S: Store>(
new_repo: &ArcRepo,
store: &S,
config: &Configuration,
identifier: &S::Identifier,
identifier: &Arc<str>,
) -> Result<Details, Error> {
if let Some(details) = new_repo.details(identifier).await? {
Ok(details)
@ -342,9 +346,9 @@ async fn fetch_or_generate_details<S: Store>(
old_repo: &OldSledRepo,
store: &S,
config: &Configuration,
identifier: &S::Identifier,
identifier: &Arc<str>,
) -> Result<Details, Error> {
let details_opt = old_repo.details(identifier).await?;
let details_opt = old_repo.details(identifier.clone()).await?;
if let Some(details) = details_opt {
Ok(details)

1854
src/repo/postgres.rs Normal file
View File

File diff suppressed because it is too large Load Diff

3
src/repo/postgres/embedded.rs Normal file
View File

@ -0,0 +1,3 @@
use refinery::embed_migrations;
embed_migrations!("./src/repo/postgres/migrations");

6
src/repo/postgres/job_status.rs Normal file
View File

@ -0,0 +1,6 @@
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, diesel_derive_enum::DbEnum)]
#[ExistingTypePath = "crate::repo::postgres::schema::sql_types::JobStatus"]
pub(super) enum JobStatus {
New,
Running,
}

11
src/repo/postgres/migrations/V0000__enable_pgcrypto.rs Normal file
View File

@ -0,0 +1,11 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.inject_custom("CREATE EXTENSION IF NOT EXISTS pgcrypto;");
m.make::<Pg>().to_string()
}

33
src/repo/postgres/migrations/V0001__create_hashes.rs Normal file
View File

@ -0,0 +1,33 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("hashes", |t| {
t.add_column(
"hash",
types::text()
.primary(true)
.unique(true)
.nullable(false)
.size(128),
);
t.add_column("identifier", types::text().unique(true).nullable(false));
t.add_column(
"motion_identifier",
types::text().unique(true).nullable(true),
);
t.add_column(
"created_at",
types::datetime()
.nullable(false)
.default(AutogenFunction::CurrentTimestamp),
);
t.add_index("ordered_hash_index", types::index(["created_at", "hash"]));
});
m.make::<Pg>().to_string()
}

28
src/repo/postgres/migrations/V0002__create_variants.rs Normal file
View File

@ -0,0 +1,28 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("variants", |t| {
t.inject_custom(r#""id" UUID PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL UNIQUE"#);
t.add_column("hash", types::text().nullable(false));
t.add_column("variant", types::text().nullable(false));
t.add_column("identifier", types::text().nullable(false));
t.add_column(
"accessed",
types::datetime()
.nullable(false)
.default(AutogenFunction::CurrentTimestamp),
);
t.add_foreign_key(&["hash"], "hashes", &["hash"]);
t.add_index(
"hash_variant_index",
types::index(["hash", "variant"]).unique(true),
);
});
m.make::<Pg>().to_string()
}

25
src/repo/postgres/migrations/V0003__create_aliases.rs Normal file
View File

@ -0,0 +1,25 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("aliases", |t| {
t.add_column(
"alias",
types::text()
.size(60)
.primary(true)
.unique(true)
.nullable(false),
);
t.add_column("hash", types::text().nullable(false));
t.add_column("token", types::text().size(60).nullable(false));
t.add_foreign_key(&["hash"], "hashes", &["hash"]);
t.add_index("aliases_hash_index", types::index(["hash"]));
});
m.make::<Pg>().to_string()
}

21
src/repo/postgres/migrations/V0004__create_settings.rs Normal file
View File

@ -0,0 +1,21 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("settings", |t| {
t.add_column(
"key",
types::text()
.size(80)
.primary(true)
.unique(true)
.nullable(false),
);
t.add_column("value", types::text().size(80).nullable(false));
});
m.make::<Pg>().to_string()
}

17
src/repo/postgres/migrations/V0005__create_details.rs Normal file
View File

@ -0,0 +1,17 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("details", |t| {
t.add_column(
"identifier",
types::text().primary(true).unique(true).nullable(false),
);
t.add_column("json", types::custom("jsonb").nullable(false));
});
m.make::<Pg>().to_string()
}

57
src/repo/postgres/migrations/V0006__create_queue.rs Normal file
View File

@ -0,0 +1,57 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.inject_custom("CREATE TYPE job_status AS ENUM ('new', 'running');");
m.create_table("job_queue", |t| {
t.inject_custom(r#""id" UUID PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL UNIQUE"#);
t.add_column("queue", types::text().size(50).nullable(false));
t.add_column("job", types::custom("jsonb").nullable(false));
t.add_column("worker", types::uuid().nullable(true));
t.add_column(
"status",
types::custom("job_status").nullable(false).default("new"),
);
t.add_column(
"queue_time",
types::datetime()
.nullable(false)
.default(AutogenFunction::CurrentTimestamp),
);
t.add_column("heartbeat", types::datetime().nullable(true));
t.add_index("queue_status_index", types::index(["queue", "status"]));
t.add_index("heartbeat_index", types::index(["heartbeat"]));
});
m.inject_custom(
r#"
CREATE OR REPLACE FUNCTION queue_status_notify()
RETURNS trigger AS
$$
BEGIN
PERFORM pg_notify('queue_status_channel', NEW.id::text || ' ' || NEW.queue::text);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
"#
.trim(),
);
m.inject_custom(
r#"
CREATE TRIGGER queue_status
AFTER INSERT OR UPDATE OF status
ON job_queue
FOR EACH ROW
EXECUTE PROCEDURE queue_status_notify();
"#
.trim(),
);
m.make::<Pg>().to_string()
}

17
src/repo/postgres/migrations/V0007__create_store_migrations.rs Normal file
View File

@ -0,0 +1,17 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("store_migrations", |t| {
t.add_column(
"old_identifier",
types::text().primary(true).nullable(false).unique(true),
);
t.add_column("new_identifier", types::text().nullable(false).unique(true));
});
m.make::<Pg>().to_string()
}

25
src/repo/postgres/migrations/V0008__create_proxies.rs Normal file
View File

@ -0,0 +1,25 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("proxies", |t| {
t.add_column(
"url",
types::text().primary(true).unique(true).nullable(false),
);
t.add_column("alias", types::text().nullable(false));
t.add_column(
"accessed",
types::datetime()
.nullable(false)
.default(AutogenFunction::CurrentTimestamp),
);
t.add_foreign_key(&["alias"], "aliases", &["alias"]);
});
m.make::<Pg>().to_string()
}

44
src/repo/postgres/migrations/V0009__create_uploads.rs Normal file
View File

@ -0,0 +1,44 @@
use barrel::backend::Pg;
use barrel::functions::AutogenFunction;
use barrel::{types, Migration};
pub(crate) fn migration() -> String {
let mut m = Migration::new();
m.create_table("uploads", |t| {
t.inject_custom(r#""id" UUID PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL UNIQUE"#);
t.add_column("result", types::custom("jsonb").nullable(true));
t.add_column(
"created_at",
types::datetime()
.nullable(false)
.default(AutogenFunction::CurrentTimestamp),
);
});
m.inject_custom(
r#"
CREATE OR REPLACE FUNCTION upload_completion_notify()
RETURNS trigger AS
$$
BEGIN
PERFORM pg_notify('upload_completion_channel', NEW.id::text);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
"#
.trim(),
);
m.inject_custom(
r#"
CREATE TRIGGER upload_result
AFTER INSERT OR UPDATE OF result
ON uploads
FOR EACH ROW
EXECUTE PROCEDURE upload_completion_notify();
"#,
);
m.make::<Pg>().to_string()
}

115
src/repo/postgres/schema.rs Normal file
View File

@ -0,0 +1,115 @@
// @generated automatically by Diesel CLI.
pub mod sql_types {
#[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)]
#[diesel(postgres_type(name = "job_status"))]
pub struct JobStatus;
}
diesel::table! {
aliases (alias) {
alias -> Text,
hash -> Text,
token -> Text,
}
}
diesel::table! {
details (identifier) {
identifier -> Text,
json -> Jsonb,
}
}
diesel::table! {
hashes (hash) {
hash -> Text,
identifier -> Text,
motion_identifier -> Nullable<Text>,
created_at -> Timestamp,
}
}
diesel::table! {
use diesel::sql_types::*;
use super::sql_types::JobStatus;
job_queue (id) {
id -> Uuid,
queue -> Text,
job -> Jsonb,
worker -> Nullable<Uuid>,
status -> JobStatus,
queue_time -> Timestamp,
heartbeat -> Nullable<Timestamp>,
}
}
diesel::table! {
proxies (url) {
url -> Text,
alias -> Text,
accessed -> Timestamp,
}
}
diesel::table! {
refinery_schema_history (version) {
version -> Int4,
#[max_length = 255]
name -> Nullable<Varchar>,
#[max_length = 255]
applied_on -> Nullable<Varchar>,
#[max_length = 255]
checksum -> Nullable<Varchar>,
}
}
diesel::table! {
settings (key) {
key -> Text,
value -> Text,
}
}
diesel::table! {
store_migrations (old_identifier) {
old_identifier -> Text,
new_identifier -> Text,
}
}
diesel::table! {
uploads (id) {
id -> Uuid,
result -> Nullable<Jsonb>,
created_at -> Timestamp,
}
}
diesel::table! {
variants (id) {
id -> Uuid,
hash -> Text,
variant -> Text,
identifier -> Text,
accessed -> Timestamp,
}
}
diesel::joinable!(aliases -> hashes (hash));
diesel::joinable!(proxies -> aliases (alias));
diesel::joinable!(variants -> hashes (hash));
diesel::allow_tables_to_appear_in_same_query!(
aliases,
details,
hashes,
job_queue,
proxies,
refinery_schema_history,
settings,
store_migrations,
uploads,
variants,
);

267
src/repo/sled.rs
View File

@ -2,7 +2,6 @@ use crate::{
details::HumanDate,
error_code::{ErrorCode, OwnedErrorCode},
serde_str::Serde,
store::StoreError,
stream::{from_iterator, LocalBoxStream},
};
use sled::{transaction::TransactionError, Db, IVec, Transactional, Tree};
@ -13,26 +12,25 @@ use std::{
atomic::{AtomicU64, Ordering},
Arc, RwLock,
},
time::Instant,
};
use tokio::sync::Notify;
use url::Url;
use uuid::Uuid;
use super::{
hash::Hash, Alias, AliasAccessRepo, AliasAlreadyExists, AliasRepo, BaseRepo, DeleteToken,
Details, DetailsRepo, FullRepo, HashAlreadyExists, HashPage, HashRepo, Identifier, JobId,
OrderedHash, ProxyRepo, QueueRepo, RepoError, SettingsRepo, StoreMigrationRepo, UploadId,
UploadRepo, UploadResult, VariantAccessRepo, VariantAlreadyExists,
hash::Hash,
metrics::{PopMetricsGuard, PushMetricsGuard, WaitMetricsGuard},
Alias, AliasAccessRepo, AliasAlreadyExists, AliasRepo, BaseRepo, DeleteToken, Details,
DetailsRepo, FullRepo, HashAlreadyExists, HashPage, HashRepo, JobId, OrderedHash, ProxyRepo,
QueueRepo, RepoError, SettingsRepo, StoreMigrationRepo, UploadId, UploadRepo, UploadResult,
VariantAccessRepo, VariantAlreadyExists,
};
macro_rules! b {
($self:ident.$ident:ident, $expr:expr) => {{
let $ident = $self.$ident.clone();
let span = tracing::Span::current();
actix_rt::task::spawn_blocking(move || span.in_scope(|| $expr))
crate::sync::spawn_blocking(move || $expr)
.await
.map_err(SledError::from)
.map_err(RepoError::from)?
@ -47,14 +45,20 @@ pub(crate) enum SledError {
Sled(#[from] sled::Error),
#[error("Invalid details json")]
Details(serde_json::Error),
Details(#[source] serde_json::Error),
#[error("Invalid upload result json")]
UploadResult(serde_json::Error),
UploadResult(#[source] serde_json::Error),
#[error("Error parsing variant key")]
VariantKey(#[from] VariantKeyError),
#[error("Invalid string data in db")]
Utf8(#[source] std::str::Utf8Error),
#[error("Invalid job json")]
Job(#[source] serde_json::Error),
#[error("Operation panicked")]
Panic,
@ -65,9 +69,10 @@ pub(crate) enum SledError {
impl SledError {
pub(super) const fn error_code(&self) -> ErrorCode {
match self {
Self::Sled(_) | Self::VariantKey(_) => ErrorCode::SLED_ERROR,
Self::Sled(_) | Self::VariantKey(_) | Self::Utf8(_) => ErrorCode::SLED_ERROR,
Self::Details(_) => ErrorCode::EXTRACT_DETAILS,
Self::UploadResult(_) => ErrorCode::EXTRACT_UPLOAD_RESULT,
Self::Job(_) => ErrorCode::EXTRACT_JOB,
Self::Panic => ErrorCode::PANIC,
Self::Conflict => ErrorCode::CONFLICTED_RECORD,
}
@ -169,7 +174,7 @@ impl SledRepo {
let this = self.db.clone();
actix_rt::task::spawn_blocking(move || {
crate::sync::spawn_blocking(move || {
let export = this.export();
export_db.import(export);
})
@ -252,7 +257,7 @@ impl AliasAccessRepo for SledRepo {
let alias_access = self.alias_access.clone();
let inverse_alias_access = self.inverse_alias_access.clone();
let res = actix_rt::task::spawn_blocking(move || {
let res = crate::sync::spawn_blocking(move || {
(&alias_access, &inverse_alias_access).transaction(
|(alias_access, inverse_alias_access)| {
if let Some(old) = alias_access.insert(alias.to_bytes(), &value_bytes)? {
@ -318,7 +323,7 @@ impl AliasAccessRepo for SledRepo {
let alias_access = self.alias_access.clone();
let inverse_alias_access = self.inverse_alias_access.clone();
let res = actix_rt::task::spawn_blocking(move || {
let res = crate::sync::spawn_blocking(move || {
(&alias_access, &inverse_alias_access).transaction(
|(alias_access, inverse_alias_access)| {
if let Some(old) = alias_access.remove(alias.to_bytes())? {
@ -358,7 +363,7 @@ impl VariantAccessRepo for SledRepo {
let variant_access = self.variant_access.clone();
let inverse_variant_access = self.inverse_variant_access.clone();
let res = actix_rt::task::spawn_blocking(move || {
let res = crate::sync::spawn_blocking(move || {
(&variant_access, &inverse_variant_access).transaction(
|(variant_access, inverse_variant_access)| {
if let Some(old) = variant_access.insert(&key, &value_bytes)? {
@ -428,7 +433,7 @@ impl VariantAccessRepo for SledRepo {
let variant_access = self.variant_access.clone();
let inverse_variant_access = self.inverse_variant_access.clone();
let res = actix_rt::task::spawn_blocking(move || {
let res = crate::sync::spawn_blocking(move || {
(&variant_access, &inverse_variant_access).transaction(
|(variant_access, inverse_variant_access)| {
if let Some(old) = variant_access.remove(&key)? {
@ -486,54 +491,6 @@ impl From<InnerUploadResult> for UploadResult {
}
}
struct PushMetricsGuard {
queue: &'static str,
armed: bool,
}
struct PopMetricsGuard {
queue: &'static str,
start: Instant,
armed: bool,
}
impl PushMetricsGuard {
fn guard(queue: &'static str) -> Self {
Self { queue, armed: true }
}
fn disarm(mut self) {
self.armed = false;
}
}
impl PopMetricsGuard {
fn guard(queue: &'static str) -> Self {
Self {
queue,
start: Instant::now(),
armed: true,
}
}
fn disarm(mut self) {
self.armed = false;
}
}
impl Drop for PushMetricsGuard {
fn drop(&mut self) {
metrics::increment_counter!("pict-rs.queue.push", "completed" => (!self.armed).to_string(), "queue" => self.queue);
}
}
impl Drop for PopMetricsGuard {
fn drop(&mut self) {
metrics::histogram!("pict-rs.queue.pop.duration", self.start.elapsed().as_secs_f64(), "completed" => (!self.armed).to_string(), "queue" => self.queue);
metrics::increment_counter!("pict-rs.queue.pop", "completed" => (!self.armed).to_string(), "queue" => self.queue);
}
}
#[async_trait::async_trait(?Send)]
impl UploadRepo for SledRepo {
#[tracing::instrument(level = "trace", skip(self))]
@ -547,6 +504,7 @@ impl UploadRepo for SledRepo {
#[tracing::instrument(skip(self))]
async fn wait(&self, upload_id: UploadId) -> Result<UploadResult, RepoError> {
let guard = WaitMetricsGuard::guard();
let mut subscriber = self.uploads.watch_prefix(upload_id.as_bytes());
let bytes = upload_id.as_bytes().to_vec();
@ -556,6 +514,7 @@ impl UploadRepo for SledRepo {
if bytes != b"1" {
let result: InnerUploadResult =
serde_json::from_slice(&bytes).map_err(SledError::UploadResult)?;
guard.disarm();
return Ok(result.into());
}
} else {
@ -571,6 +530,8 @@ impl UploadRepo for SledRepo {
if value != b"1" {
let result: InnerUploadResult =
serde_json::from_slice(&value).map_err(SledError::UploadResult)?;
guard.disarm();
return Ok(result.into());
}
}
@ -648,19 +609,31 @@ fn job_key(queue: &'static str, job_id: JobId) -> Arc<[u8]> {
Arc::from(key)
}
fn try_into_arc_str(ivec: IVec) -> Result<Arc<str>, SledError> {
std::str::from_utf8(&ivec[..])
.map_err(SledError::Utf8)
.map(String::from)
.map(Arc::from)
}
#[async_trait::async_trait(?Send)]
impl QueueRepo for SledRepo {
#[tracing::instrument(skip(self, job), fields(job = %String::from_utf8_lossy(&job)))]
async fn push(&self, queue_name: &'static str, job: Arc<[u8]>) -> Result<JobId, RepoError> {
#[tracing::instrument(skip(self))]
async fn push(
&self,
queue_name: &'static str,
job: serde_json::Value,
) -> Result<JobId, RepoError> {
let metrics_guard = PushMetricsGuard::guard(queue_name);
let id = JobId::gen();
let key = job_key(queue_name, id);
let job = serde_json::to_vec(&job).map_err(SledError::Job)?;
let queue = self.queue.clone();
let job_state = self.job_state.clone();
let res = actix_rt::task::spawn_blocking(move || {
let res = crate::sync::spawn_blocking(move || {
(&queue, &job_state).transaction(|(queue, job_state)| {
let state = JobState::pending();
@ -687,7 +660,7 @@ impl QueueRepo for SledRepo {
.write()
.unwrap()
.entry(queue_name)
.or_insert_with(|| Arc::new(Notify::new()))
.or_insert_with(crate::sync::notify)
.notify_one();
metrics_guard.disarm();
@ -700,7 +673,7 @@ impl QueueRepo for SledRepo {
&self,
queue_name: &'static str,
worker_id: Uuid,
) -> Result<(JobId, Arc<[u8]>), RepoError> {
) -> Result<(JobId, serde_json::Value), RepoError> {
let metrics_guard = PopMetricsGuard::guard(queue_name);
let now = time::OffsetDateTime::now_utc();
@ -710,7 +683,7 @@ impl QueueRepo for SledRepo {
let job_state = self.job_state.clone();
let span = tracing::Span::current();
let opt = actix_rt::task::spawn_blocking(move || {
let opt = crate::sync::spawn_blocking(move || {
let _guard = span.enter();
// Job IDs are generated with Uuid version 7 - defining their first bits as a
// timestamp. Scanning a prefix should give us jobs in the order they were queued.
@ -755,9 +728,12 @@ impl QueueRepo for SledRepo {
let opt = queue
.get(&key)?
.map(|job_bytes| (job_id, Arc::from(job_bytes.to_vec())));
.map(|ivec| serde_json::from_slice(&ivec[..]))
.transpose()
.map_err(SledError::Job)?;
return Ok(opt) as Result<Option<(JobId, Arc<[u8]>)>, SledError>;
return Ok(opt.map(|job| (job_id, job)))
as Result<Option<(JobId, serde_json::Value)>, SledError>;
}
Ok(None)
@ -781,9 +757,7 @@ impl QueueRepo for SledRepo {
notify
} else {
let mut guard = self.queue_notifier.write().unwrap();
let entry = guard
.entry(queue_name)
.or_insert_with(|| Arc::new(Notify::new()));
let entry = guard.entry(queue_name).or_insert_with(crate::sync::notify);
Arc::clone(entry)
};
@ -802,7 +776,7 @@ impl QueueRepo for SledRepo {
let job_state = self.job_state.clone();
actix_rt::task::spawn_blocking(move || {
crate::sync::spawn_blocking(move || {
if let Some(state) = job_state.get(&key)? {
let new_state = JobState::running(worker_id);
@ -832,7 +806,7 @@ impl QueueRepo for SledRepo {
let queue = self.queue.clone();
let job_state = self.job_state.clone();
let res = actix_rt::task::spawn_blocking(move || {
let res = crate::sync::spawn_blocking(move || {
(&queue, &job_state).transaction(|(queue, job_state)| {
queue.remove(&key[..])?;
job_state.remove(&key[..])?;
@ -949,43 +923,46 @@ fn variant_from_key(hash: &[u8], key: &[u8]) -> Option<String> {
#[async_trait::async_trait(?Send)]
impl DetailsRepo for SledRepo {
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
#[tracing::instrument(level = "trace", skip(self))]
async fn relate_details(
&self,
identifier: &dyn Identifier,
identifier: &Arc<str>,
details: &Details,
) -> Result<(), StoreError> {
let key = identifier.to_bytes()?;
let details = serde_json::to_vec(&details.inner)
.map_err(SledError::Details)
.map_err(RepoError::from)?;
) -> Result<(), RepoError> {
let key = identifier.clone();
let details = serde_json::to_vec(&details.inner).map_err(SledError::Details)?;
b!(
self.identifier_details,
identifier_details.insert(key, details)
identifier_details.insert(key.as_bytes(), details)
);
Ok(())
}
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
async fn details(&self, identifier: &dyn Identifier) -> Result<Option<Details>, StoreError> {
let key = identifier.to_bytes()?;
#[tracing::instrument(level = "trace", skip(self))]
async fn details(&self, identifier: &Arc<str>) -> Result<Option<Details>, RepoError> {
let key = identifier.clone();
let opt = b!(self.identifier_details, identifier_details.get(key));
let opt = b!(
self.identifier_details,
identifier_details.get(key.as_bytes())
);
opt.map(|ivec| serde_json::from_slice(&ivec).map(|inner| Details { inner }))
.transpose()
.map_err(SledError::Details)
.map_err(RepoError::from)
.map_err(StoreError::from)
}
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
async fn cleanup_details(&self, identifier: &dyn Identifier) -> Result<(), StoreError> {
let key = identifier.to_bytes()?;
#[tracing::instrument(level = "trace", skip(self))]
async fn cleanup_details(&self, identifier: &Arc<str>) -> Result<(), RepoError> {
let key = identifier.clone();
b!(self.identifier_details, identifier_details.remove(key));
b!(
self.identifier_details,
identifier_details.remove(key.as_bytes())
);
Ok(())
}
@ -999,24 +976,28 @@ impl StoreMigrationRepo for SledRepo {
async fn mark_migrated(
&self,
old_identifier: &dyn Identifier,
new_identifier: &dyn Identifier,
) -> Result<(), StoreError> {
let key = new_identifier.to_bytes()?;
let value = old_identifier.to_bytes()?;
old_identifier: &Arc<str>,
new_identifier: &Arc<str>,
) -> Result<(), RepoError> {
let key = new_identifier.clone();
let value = old_identifier.clone();
b!(
self.migration_identifiers,
migration_identifiers.insert(key, value)
migration_identifiers.insert(key.as_bytes(), value.as_bytes())
);
Ok(())
}
async fn is_migrated(&self, identifier: &dyn Identifier) -> Result<bool, StoreError> {
let key = identifier.to_bytes()?;
async fn is_migrated(&self, identifier: &Arc<str>) -> Result<bool, RepoError> {
let key = identifier.clone();
Ok(b!(self.migration_identifiers, migration_identifiers.get(key)).is_some())
Ok(b!(
self.migration_identifiers,
migration_identifiers.get(key.as_bytes())
)
.is_some())
}
async fn clear(&self) -> Result<(), RepoError> {
@ -1062,17 +1043,6 @@ impl HashRepo for SledRepo {
))
}
async fn hashes(&self) -> LocalBoxStream<'static, Result<Hash, RepoError>> {
let iter = self.hashes.iter().keys().filter_map(|res| {
res.map_err(SledError::from)
.map_err(RepoError::from)
.map(Hash::from_ivec)
.transpose()
});
Box::pin(from_iterator(iter, 8))
}
async fn bound(&self, hash: Hash) -> Result<Option<OrderedHash>, RepoError> {
let opt = b!(self.hashes, hashes.get(hash.to_ivec()));
@ -1095,7 +1065,7 @@ impl HashRepo for SledRepo {
None => (self.hashes_inverse.iter(), None),
};
actix_rt::task::spawn_blocking(move || {
crate::sync::spawn_blocking(move || {
let page_iter = page_iter
.keys()
.rev()
@ -1147,7 +1117,7 @@ impl HashRepo for SledRepo {
let page_iter = self.hashes_inverse.range(..=date_nanos);
let prev_iter = Some(self.hashes_inverse.range(date_nanos..));
actix_rt::task::spawn_blocking(move || {
crate::sync::spawn_blocking(move || {
let page_iter = page_iter
.keys()
.rev()
@ -1197,10 +1167,10 @@ impl HashRepo for SledRepo {
async fn create_hash_with_timestamp(
&self,
hash: Hash,
identifier: &dyn Identifier,
identifier: &Arc<str>,
timestamp: time::OffsetDateTime,
) -> Result<Result<(), HashAlreadyExists>, StoreError> {
let identifier: sled::IVec = identifier.to_bytes()?.into();
) -> Result<Result<(), HashAlreadyExists>, RepoError> {
let identifier: sled::IVec = identifier.as_bytes().to_vec().into();
let hashes = self.hashes.clone();
let hashes_inverse = self.hashes_inverse.clone();
@ -1234,63 +1204,56 @@ impl HashRepo for SledRepo {
match res {
Ok(res) => Ok(res),
Err(TransactionError::Abort(e) | TransactionError::Storage(e)) => {
Err(StoreError::from(RepoError::from(SledError::from(e))))
Err(RepoError::from(SledError::from(e)))
}
}
}
async fn update_identifier(
&self,
hash: Hash,
identifier: &dyn Identifier,
) -> Result<(), StoreError> {
let identifier = identifier.to_bytes()?;
async fn update_identifier(&self, hash: Hash, identifier: &Arc<str>) -> Result<(), RepoError> {
let identifier = identifier.clone();
let hash = hash.to_ivec();
b!(
self.hash_identifiers,
hash_identifiers.insert(hash, identifier)
hash_identifiers.insert(hash, identifier.as_bytes())
);
Ok(())
}
#[tracing::instrument(level = "trace", skip(self))]
async fn identifier(&self, hash: Hash) -> Result<Option<Arc<[u8]>>, RepoError> {
async fn identifier(&self, hash: Hash) -> Result<Option<Arc<str>>, RepoError> {
let hash = hash.to_ivec();
let Some(ivec) = b!(self.hash_identifiers, hash_identifiers.get(hash)) else {
return Ok(None);
};
let opt = b!(self.hash_identifiers, hash_identifiers.get(hash));
Ok(Some(Arc::from(ivec.to_vec())))
Ok(opt.map(try_into_arc_str).transpose()?)
}
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
#[tracing::instrument(level = "trace", skip(self))]
async fn relate_variant_identifier(
&self,
hash: Hash,
variant: String,
identifier: &dyn Identifier,
) -> Result<Result<(), VariantAlreadyExists>, StoreError> {
identifier: &Arc<str>,
) -> Result<Result<(), VariantAlreadyExists>, RepoError> {
let hash = hash.to_bytes();
let key = variant_key(&hash, &variant);
let value = identifier.to_bytes()?;
let value = identifier.clone();
let hash_variant_identifiers = self.hash_variant_identifiers.clone();
actix_rt::task::spawn_blocking(move || {
crate::sync::spawn_blocking(move || {
hash_variant_identifiers
.compare_and_swap(key, Option::<&[u8]>::None, Some(value))
.compare_and_swap(key, Option::<&[u8]>::None, Some(value.as_bytes()))
.map(|res| res.map_err(|_| VariantAlreadyExists))
})
.await
.map_err(|_| RepoError::Canceled)?
.map_err(SledError::from)
.map_err(RepoError::from)
.map_err(StoreError::from)
}
#[tracing::instrument(level = "trace", skip(self))]
@ -1298,7 +1261,7 @@ impl HashRepo for SledRepo {
&self,
hash: Hash,
variant: String,
) -> Result<Option<Arc<[u8]>>, RepoError> {
) -> Result<Option<Arc<str>>, RepoError> {
let hash = hash.to_bytes();
let key = variant_key(&hash, &variant);
@ -1308,11 +1271,11 @@ impl HashRepo for SledRepo {
hash_variant_identifiers.get(key)
);
Ok(opt.map(|ivec| Arc::from(ivec.to_vec())))
Ok(opt.map(try_into_arc_str).transpose()?)
}
#[tracing::instrument(level = "debug", skip(self))]
async fn variants(&self, hash: Hash) -> Result<Vec<(String, Arc<[u8]>)>, RepoError> {
async fn variants(&self, hash: Hash) -> Result<Vec<(String, Arc<str>)>, RepoError> {
let hash = hash.to_ivec();
let vec = b!(
@ -1321,14 +1284,14 @@ impl HashRepo for SledRepo {
.scan_prefix(hash.clone())
.filter_map(|res| res.ok())
.filter_map(|(key, ivec)| {
let identifier = Arc::from(ivec.to_vec());
let identifier = try_into_arc_str(ivec).ok();
let variant = variant_from_key(&hash, &key);
if variant.is_none() {
tracing::warn!("Skipping a variant: {}", String::from_utf8_lossy(&key));
}
Some((variant?, identifier))
Some((variant?, identifier?))
})
.collect::<Vec<_>>()) as Result<Vec<_>, SledError>
);
@ -1350,25 +1313,25 @@ impl HashRepo for SledRepo {
Ok(())
}
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
#[tracing::instrument(level = "trace", skip(self))]
async fn relate_motion_identifier(
&self,
hash: Hash,
identifier: &dyn Identifier,
) -> Result<(), StoreError> {
identifier: &Arc<str>,
) -> Result<(), RepoError> {
let hash = hash.to_ivec();
let bytes = identifier.to_bytes()?;
let bytes = identifier.clone();
b!(
self.hash_motion_identifiers,
hash_motion_identifiers.insert(hash, bytes)
hash_motion_identifiers.insert(hash, bytes.as_bytes())
);
Ok(())
}
#[tracing::instrument(level = "trace", skip(self))]
async fn motion_identifier(&self, hash: Hash) -> Result<Option<Arc<[u8]>>, RepoError> {
async fn motion_identifier(&self, hash: Hash) -> Result<Option<Arc<str>>, RepoError> {
let hash = hash.to_ivec();
let opt = b!(
@ -1376,7 +1339,7 @@ impl HashRepo for SledRepo {
hash_motion_identifiers.get(hash)
);
Ok(opt.map(|ivec| Arc::from(ivec.to_vec())))
Ok(opt.map(try_into_arc_str).transpose()?)
}
#[tracing::instrument(skip(self))]

20
src/repo_04.rs
View File

@ -2,10 +2,9 @@ use crate::{
config,
details::Details,
repo::{Alias, DeleteToken},
store::{Identifier, StoreError},
};
use futures_core::Stream;
use std::fmt::Debug;
use std::{fmt::Debug, sync::Arc};
pub(crate) use self::sled::SledRepo;
@ -46,7 +45,7 @@ pub(crate) trait SettingsRepo: BaseRepo {
#[async_trait::async_trait(?Send)]
pub(crate) trait IdentifierRepo: BaseRepo {
async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, StoreError>;
async fn details(&self, identifier: Arc<str>) -> Result<Option<Details>, RepoError>;
}
#[async_trait::async_trait(?Send)]
@ -57,20 +56,11 @@ pub(crate) trait HashRepo: BaseRepo {
async fn hashes(&self) -> Self::Stream;
async fn identifier<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<Option<I>, StoreError>;
async fn identifier(&self, hash: Self::Bytes) -> Result<Option<Arc<str>>, RepoError>;
async fn variants<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<Vec<(String, I)>, StoreError>;
async fn variants(&self, hash: Self::Bytes) -> Result<Vec<(String, Arc<str>)>, RepoError>;
async fn motion_identifier<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<Option<I>, StoreError>;
async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<Arc<str>>, RepoError>;
}
#[async_trait::async_trait(?Send)]

58
src/repo_04/sled.rs
View File

@ -1,10 +1,9 @@
use crate::{
details::HumanDate,
repo_04::{
Alias, AliasRepo, BaseRepo, DeleteToken, Details, HashRepo, Identifier, IdentifierRepo,
RepoError, SettingsRepo,
Alias, AliasRepo, BaseRepo, DeleteToken, Details, HashRepo, IdentifierRepo, RepoError,
SettingsRepo,
},
store::StoreError,
stream::{from_iterator, LocalBoxStream},
};
use sled::{Db, IVec, Tree};
@ -35,9 +34,7 @@ macro_rules! b {
($self:ident.$ident:ident, $expr:expr) => {{
let $ident = $self.$ident.clone();
let span = tracing::Span::current();
actix_rt::task::spawn_blocking(move || span.in_scope(|| $expr))
crate::sync::spawn_blocking(move || $expr)
.await
.map_err(SledError::from)
.map_err(RepoError::from)?
@ -56,6 +53,9 @@ pub(crate) enum SledError {
#[error("Operation panicked")]
Panic,
#[error("Error reading string")]
Utf8(#[from] std::str::Utf8Error),
}
#[derive(Clone)]
@ -179,17 +179,17 @@ fn variant_from_key(hash: &[u8], key: &[u8]) -> Option<String> {
#[async_trait::async_trait(?Send)]
impl IdentifierRepo for SledRepo {
#[tracing::instrument(level = "trace", skip(self, identifier), fields(identifier = identifier.string_repr()))]
async fn details<I: Identifier>(&self, identifier: &I) -> Result<Option<Details>, StoreError> {
let key = identifier.to_bytes()?;
let opt = b!(self.identifier_details, identifier_details.get(key));
#[tracing::instrument(level = "trace", skip(self))]
async fn details(&self, key: Arc<str>) -> Result<Option<Details>, RepoError> {
let opt = b!(
self.identifier_details,
identifier_details.get(key.as_bytes())
);
opt.map(|ivec| serde_json::from_slice::<OldDetails>(&ivec))
.transpose()
.map_err(SledError::from)
.map_err(RepoError::from)
.map_err(StoreError::from)
.map(|opt| opt.and_then(OldDetails::into_details))
}
}
@ -219,29 +219,27 @@ impl HashRepo for SledRepo {
}
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn identifier<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<Option<I>, StoreError> {
async fn identifier(&self, hash: Self::Bytes) -> Result<Option<Arc<str>>, RepoError> {
let Some(ivec) = b!(self.hash_identifiers, hash_identifiers.get(hash)) else {
return Ok(None);
};
Ok(Some(I::from_bytes(ivec.to_vec())?))
Ok(Some(Arc::from(
std::str::from_utf8(&ivec[..])
.map_err(SledError::from)?
.to_string(),
)))
}
#[tracing::instrument(level = "debug", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn variants<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<Vec<(String, I)>, StoreError> {
async fn variants(&self, hash: Self::Bytes) -> Result<Vec<(String, Arc<str>)>, RepoError> {
let vec = b!(
self.hash_variant_identifiers,
Ok(hash_variant_identifiers
.scan_prefix(&hash)
.filter_map(|res| res.ok())
.filter_map(|(key, ivec)| {
let identifier = I::from_bytes(ivec.to_vec()).ok();
let identifier = String::from_utf8(ivec.to_vec()).ok();
if identifier.is_none() {
tracing::warn!(
"Skipping an identifier: {}",
@ -254,7 +252,7 @@ impl HashRepo for SledRepo {
tracing::warn!("Skipping a variant: {}", String::from_utf8_lossy(&key));
}
Some((variant?, identifier?))
Some((variant?, Arc::from(identifier?)))
})
.collect::<Vec<_>>()) as Result<Vec<_>, SledError>
);
@ -263,16 +261,20 @@ impl HashRepo for SledRepo {
}
#[tracing::instrument(level = "trace", skip(self, hash), fields(hash = hex::encode(&hash)))]
async fn motion_identifier<I: Identifier + 'static>(
&self,
hash: Self::Bytes,
) -> Result<Option<I>, StoreError> {
async fn motion_identifier(&self, hash: Self::Bytes) -> Result<Option<Arc<str>>, RepoError> {
let opt = b!(
self.hash_motion_identifiers,
hash_motion_identifiers.get(hash)
);
opt.map(|ivec| I::from_bytes(ivec.to_vec())).transpose()
opt.map(|ivec| {
Ok(Arc::from(
std::str::from_utf8(&ivec[..])
.map_err(SledError::from)?
.to_string(),
))
})
.transpose()
}
}

208
src/store.rs
View File

@ -1,10 +1,9 @@
use actix_web::web::Bytes;
use base64::{prelude::BASE64_STANDARD, Engine};
use futures_core::Stream;
use std::{fmt::Debug, sync::Arc};
use tokio::io::{AsyncRead, AsyncWrite};
use crate::error_code::ErrorCode;
use crate::{error_code::ErrorCode, stream::LocalBoxStream};
pub(crate) mod file_store;
pub(crate) mod object_store;
@ -40,9 +39,17 @@ impl StoreError {
Self::FileNotFound(_) | Self::ObjectNotFound(_) => ErrorCode::NOT_FOUND,
}
}
pub(crate) const fn is_not_found(&self) -> bool {
matches!(self, Self::FileNotFound(_)) || matches!(self, Self::ObjectNotFound(_))
}
pub(crate) const fn is_disconnected(&self) -> bool {
match self {
Self::Repo(e) => e.is_disconnected(),
_ => false,
}
}
}
impl From<crate::store::file_store::FileError> for StoreError {
@ -70,32 +77,15 @@ impl From<crate::store::object_store::ObjectError> for StoreError {
}
}
pub(crate) trait Identifier: Send + Sync + Debug {
fn to_bytes(&self) -> Result<Vec<u8>, StoreError>;
fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError>
where
Self: Sized;
fn from_arc(arc: Arc<[u8]>) -> Result<Self, StoreError>
where
Self: Sized;
fn string_repr(&self) -> String;
}
#[async_trait::async_trait(?Send)]
pub(crate) trait Store: Clone + Debug {
type Identifier: Identifier + Clone + 'static;
type Stream: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static;
async fn health_check(&self) -> Result<(), StoreError>;
async fn save_async_read<Reader>(
&self,
reader: Reader,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead + Unpin + 'static;
@ -103,7 +93,7 @@ pub(crate) trait Store: Clone + Debug {
&self,
stream: S,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static;
@ -111,28 +101,28 @@ pub(crate) trait Store: Clone + Debug {
&self,
bytes: Bytes,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>;
) -> Result<Arc<str>, StoreError>;
fn public_url(&self, _: &Self::Identifier) -> Option<url::Url>;
fn public_url(&self, _: &Arc<str>) -> Option<url::Url>;
async fn to_stream(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
from_start: Option<u64>,
len: Option<u64>,
) -> Result<Self::Stream, StoreError>;
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError>;
async fn read_into<Writer>(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
writer: &mut Writer,
) -> Result<(), std::io::Error>
where
Writer: AsyncWrite + Unpin;
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError>;
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError>;
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError>;
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError>;
}
#[async_trait::async_trait(?Send)]
@ -140,9 +130,6 @@ impl<T> Store for actix_web::web::Data<T>
where
T: Store,
{
type Identifier = T::Identifier;
type Stream = T::Stream;
async fn health_check(&self) -> Result<(), StoreError> {
T::health_check(self).await
}
@ -151,7 +138,7 @@ where
&self,
reader: Reader,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead + Unpin + 'static,
{
@ -162,7 +149,7 @@ where
&self,
stream: S,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{
@ -173,26 +160,26 @@ where
&self,
bytes: Bytes,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError> {
) -> Result<Arc<str>, StoreError> {
T::save_bytes(self, bytes, content_type).await
}
fn public_url(&self, identifier: &Self::Identifier) -> Option<url::Url> {
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
T::public_url(self, identifier)
}
async fn to_stream(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
from_start: Option<u64>,
len: Option<u64>,
) -> Result<Self::Stream, StoreError> {
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
T::to_stream(self, identifier, from_start, len).await
}
async fn read_into<Writer>(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
writer: &mut Writer,
) -> Result<(), std::io::Error>
where
@ -201,11 +188,83 @@ where
T::read_into(self, identifier, writer).await
}
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
T::len(self, identifier).await
}
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
T::remove(self, identifier).await
}
}
#[async_trait::async_trait(?Send)]
impl<T> Store for Arc<T>
where
T: Store,
{
async fn health_check(&self) -> Result<(), StoreError> {
T::health_check(self).await
}
async fn save_async_read<Reader>(
&self,
reader: Reader,
content_type: mime::Mime,
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead + Unpin + 'static,
{
T::save_async_read(self, reader, content_type).await
}
async fn save_stream<S>(
&self,
stream: S,
content_type: mime::Mime,
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{
T::save_stream(self, stream, content_type).await
}
async fn save_bytes(
&self,
bytes: Bytes,
content_type: mime::Mime,
) -> Result<Arc<str>, StoreError> {
T::save_bytes(self, bytes, content_type).await
}
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
T::public_url(self, identifier)
}
async fn to_stream(
&self,
identifier: &Arc<str>,
from_start: Option<u64>,
len: Option<u64>,
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
T::to_stream(self, identifier, from_start, len).await
}
async fn read_into<Writer>(
&self,
identifier: &Arc<str>,
writer: &mut Writer,
) -> Result<(), std::io::Error>
where
Writer: AsyncWrite + Unpin,
{
T::read_into(self, identifier, writer).await
}
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
T::len(self, identifier).await
}
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
T::remove(self, identifier).await
}
}
@ -215,9 +274,6 @@ impl<'a, T> Store for &'a T
where
T: Store,
{
type Identifier = T::Identifier;
type Stream = T::Stream;
async fn health_check(&self) -> Result<(), StoreError> {
T::health_check(self).await
}
@ -226,7 +282,7 @@ where
&self,
reader: Reader,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead + Unpin + 'static,
{
@ -237,7 +293,7 @@ where
&self,
stream: S,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{
@ -248,26 +304,26 @@ where
&self,
bytes: Bytes,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError> {
) -> Result<Arc<str>, StoreError> {
T::save_bytes(self, bytes, content_type).await
}
fn public_url(&self, identifier: &Self::Identifier) -> Option<url::Url> {
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
T::public_url(self, identifier)
}
async fn to_stream(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
from_start: Option<u64>,
len: Option<u64>,
) -> Result<Self::Stream, StoreError> {
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
T::to_stream(self, identifier, from_start, len).await
}
async fn read_into<Writer>(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
writer: &mut Writer,
) -> Result<(), std::io::Error>
where
@ -276,59 +332,11 @@ where
T::read_into(self, identifier, writer).await
}
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
T::len(self, identifier).await
}
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
T::remove(self, identifier).await
}
}
impl Identifier for Vec<u8> {
fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError>
where
Self: Sized,
{
Ok(bytes)
}
fn from_arc(arc: Arc<[u8]>) -> Result<Self, StoreError>
where
Self: Sized,
{
Ok(Vec::from(&arc[..]))
}
fn to_bytes(&self) -> Result<Vec<u8>, StoreError> {
Ok(self.clone())
}
fn string_repr(&self) -> String {
BASE64_STANDARD.encode(self.as_slice())
}
}
impl Identifier for Arc<[u8]> {
fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError>
where
Self: Sized,
{
Ok(Arc::from(bytes))
}
fn from_arc(arc: Arc<[u8]>) -> Result<Self, StoreError>
where
Self: Sized,
{
Ok(arc)
}
fn to_bytes(&self) -> Result<Vec<u8>, StoreError> {
Ok(Vec::from(&self[..]))
}
fn string_repr(&self) -> String {
BASE64_STANDARD.encode(&self[..])
}
}

100
src/store/file_store.rs
View File

@ -1,23 +1,17 @@
use crate::{
error_code::ErrorCode,
file::File,
repo::{Repo, SettingsRepo},
store::Store,
error_code::ErrorCode, file::File, repo::ArcRepo, store::Store, stream::LocalBoxStream,
};
use actix_web::web::Bytes;
use futures_core::Stream;
use std::{
path::{Path, PathBuf},
pin::Pin,
sync::Arc,
};
use storage_path_generator::Generator;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio_util::io::StreamReader;
use tracing::Instrument;
mod file_id;
pub(crate) use file_id::FileId;
use super::StoreError;
// - Settings Tree
@ -33,12 +27,12 @@ pub(crate) enum FileError {
#[error("Failed to generate path")]
PathGenerator(#[from] storage_path_generator::PathError),
#[error("Error formatting file store ID")]
IdError,
#[error("Malformed file store ID")]
#[error("Couldn't strip root dir")]
PrefixError,
#[error("Couldn't convert Path to String")]
StringError,
#[error("Tried to save over existing file")]
FileExists,
}
@ -49,7 +43,7 @@ impl FileError {
Self::Io(_) => ErrorCode::FILE_IO_ERROR,
Self::PathGenerator(_) => ErrorCode::PARSE_PATH_ERROR,
Self::FileExists => ErrorCode::FILE_EXISTS,
Self::IdError | Self::PrefixError => ErrorCode::FORMAT_FILE_ID_ERROR,
Self::StringError | Self::PrefixError => ErrorCode::FORMAT_FILE_ID_ERROR,
}
}
}
@ -58,14 +52,12 @@ impl FileError {
pub(crate) struct FileStore {
path_gen: Generator,
root_dir: PathBuf,
repo: Repo,
repo: ArcRepo,
}
#[async_trait::async_trait(?Send)]
impl Store for FileStore {
type Identifier = FileId;
type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
#[tracing::instrument(level = "DEBUG", skip(self))]
async fn health_check(&self) -> Result<(), StoreError> {
tokio::fs::metadata(&self.root_dir)
.await
@ -74,12 +66,12 @@ impl Store for FileStore {
Ok(())
}
#[tracing::instrument(skip(reader))]
#[tracing::instrument(skip(self, reader))]
async fn save_async_read<Reader>(
&self,
mut reader: Reader,
_content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead + Unpin + 'static,
{
@ -97,7 +89,7 @@ impl Store for FileStore {
&self,
stream: S,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{
@ -105,12 +97,12 @@ impl Store for FileStore {
.await
}
#[tracing::instrument(skip(bytes))]
#[tracing::instrument(skip(self, bytes))]
async fn save_bytes(
&self,
bytes: Bytes,
_content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError> {
) -> Result<Arc<str>, StoreError> {
let path = self.next_file().await?;
if let Err(e) = self.safe_save_bytes(&path, bytes).await {
@ -121,17 +113,17 @@ impl Store for FileStore {
Ok(self.file_id_from_path(path)?)
}
fn public_url(&self, _identifier: &Self::Identifier) -> Option<url::Url> {
fn public_url(&self, _identifier: &Arc<str>) -> Option<url::Url> {
None
}
#[tracing::instrument]
#[tracing::instrument(skip(self))]
async fn to_stream(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
from_start: Option<u64>,
len: Option<u64>,
) -> Result<Self::Stream, StoreError> {
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
let path = self.path_from_file_id(identifier);
let file_span = tracing::trace_span!(parent: None, "File Stream");
@ -149,10 +141,10 @@ impl Store for FileStore {
Ok(Box::pin(stream))
}
#[tracing::instrument(skip(writer))]
#[tracing::instrument(skip(self, writer))]
async fn read_into<Writer>(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
writer: &mut Writer,
) -> Result<(), std::io::Error>
where
@ -165,8 +157,8 @@ impl Store for FileStore {
Ok(())
}
#[tracing::instrument]
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
#[tracing::instrument(skip(self))]
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
let path = self.path_from_file_id(identifier);
let len = tokio::fs::metadata(path)
@ -177,8 +169,8 @@ impl Store for FileStore {
Ok(len)
}
#[tracing::instrument]
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
#[tracing::instrument(skip(self))]
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
let path = self.path_from_file_id(identifier);
self.safe_remove_file(path).await?;
@ -189,7 +181,7 @@ impl Store for FileStore {
impl FileStore {
#[tracing::instrument(skip(repo))]
pub(crate) async fn build(root_dir: PathBuf, repo: Repo) -> color_eyre::Result<Self> {
pub(crate) async fn build(root_dir: PathBuf, repo: ArcRepo) -> color_eyre::Result<Self> {
let path_gen = init_generator(&repo).await?;
tokio::fs::create_dir_all(&root_dir).await?;
@ -201,16 +193,24 @@ impl FileStore {
})
}
fn file_id_from_path(&self, path: PathBuf) -> Result<Arc<str>, FileError> {
path.strip_prefix(&self.root_dir)
.map_err(|_| FileError::PrefixError)?
.to_str()
.ok_or(FileError::StringError)
.map(Into::into)
}
fn path_from_file_id(&self, file_id: &Arc<str>) -> PathBuf {
self.root_dir.join(file_id.as_ref())
}
async fn next_directory(&self) -> Result<PathBuf, StoreError> {
let path = self.path_gen.next();
match self.repo {
Repo::Sled(ref sled_repo) => {
sled_repo
.set(GENERATOR_KEY, path.to_be_bytes().into())
.await?;
}
}
self.repo
.set(GENERATOR_KEY, path.to_be_bytes().into())
.await?;
let mut target_path = self.root_dir.clone();
for dir in path.to_strings() {
@ -227,6 +227,7 @@ impl FileStore {
Ok(target_path.join(filename))
}
#[tracing::instrument(level = "DEBUG", skip(self, path), fields(path = ?path.as_ref()))]
async fn safe_remove_file<P: AsRef<Path>>(&self, path: P) -> Result<(), FileError> {
tokio::fs::remove_file(&path).await?;
self.try_remove_parents(path.as_ref()).await;
@ -308,18 +309,13 @@ pub(crate) async fn safe_create_parent<P: AsRef<Path>>(path: P) -> Result<(), Fi
Ok(())
}
async fn init_generator(repo: &Repo) -> Result<Generator, StoreError> {
match repo {
Repo::Sled(sled_repo) => {
if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
Ok(Generator::from_existing(
storage_path_generator::Path::from_be_bytes(ivec.to_vec())
.map_err(FileError::from)?,
))
} else {
Ok(Generator::new())
}
}
async fn init_generator(repo: &ArcRepo) -> Result<Generator, StoreError> {
if let Some(ivec) = repo.get(GENERATOR_KEY).await? {
Ok(Generator::from_existing(
storage_path_generator::Path::from_be_bytes(ivec.to_vec()).map_err(FileError::from)?,
))
} else {
Ok(Generator::new())
}
}

57
src/store/file_store/file_id.rs
View File

@ -1,57 +0,0 @@
use crate::store::{
file_store::{FileError, FileStore},
Identifier, StoreError,
};
use std::path::PathBuf;
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct FileId(PathBuf);
impl Identifier for FileId {
fn to_bytes(&self) -> Result<Vec<u8>, StoreError> {
let vec = self
.0
.to_str()
.ok_or(FileError::IdError)?
.as_bytes()
.to_vec();
Ok(vec)
}
fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError>
where
Self: Sized,
{
let string = String::from_utf8(bytes).map_err(|_| FileError::IdError)?;
let id = FileId(PathBuf::from(string));
Ok(id)
}
fn from_arc(arc: std::sync::Arc<[u8]>) -> Result<Self, StoreError>
where
Self: Sized,
{
Self::from_bytes(Vec::from(&arc[..]))
}
fn string_repr(&self) -> String {
self.0.to_string_lossy().into_owned()
}
}
impl FileStore {
pub(super) fn file_id_from_path(&self, path: PathBuf) -> Result<FileId, FileError> {
let stripped = path
.strip_prefix(&self.root_dir)
.map_err(|_| FileError::PrefixError)?;
Ok(FileId(stripped.to_path_buf()))
}
pub(super) fn path_from_file_id(&self, file_id: &FileId) -> PathBuf {
self.root_dir.join(&file_id.0)
}
}

102
src/store/object_store.rs
View File

@ -1,9 +1,9 @@
use crate::{
bytes_stream::BytesStream,
error_code::ErrorCode,
repo::{Repo, SettingsRepo},
repo::ArcRepo,
store::Store,
stream::{IntoStreamer, StreamMap},
stream::{IntoStreamer, LocalBoxStream, StreamMap},
};
use actix_rt::task::JoinError;
use actix_web::{
@ -19,16 +19,13 @@ use futures_core::Stream;
use reqwest::{header::RANGE, Body, Response};
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
use rusty_s3::{actions::S3Action, Bucket, BucketError, Credentials, UrlStyle};
use std::{pin::Pin, string::FromUtf8Error, time::Duration};
use std::{string::FromUtf8Error, sync::Arc, time::Duration};
use storage_path_generator::{Generator, Path};
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
use tokio_util::io::ReaderStream;
use tracing::Instrument;
use url::Url;
mod object_id;
pub(crate) use object_id::ObjectId;
use super::StoreError;
const CHUNK_SIZE: usize = 8_388_608; // 8 Mebibytes, min is 5 (5_242_880);
@ -107,7 +104,7 @@ impl From<BlockingError> for ObjectError {
#[derive(Clone)]
pub(crate) struct ObjectStore {
path_gen: Generator,
repo: Repo,
repo: ArcRepo,
bucket: Bucket,
credentials: Credentials,
client: ClientWithMiddleware,
@ -119,7 +116,7 @@ pub(crate) struct ObjectStore {
#[derive(Clone)]
pub(crate) struct ObjectStoreConfig {
path_gen: Generator,
repo: Repo,
repo: ArcRepo,
bucket: Bucket,
credentials: Credentials,
signature_expiration: u64,
@ -189,9 +186,6 @@ async fn status_error(response: Response) -> StoreError {
#[async_trait::async_trait(?Send)]
impl Store for ObjectStore {
type Identifier = ObjectId;
type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
async fn health_check(&self) -> Result<(), StoreError> {
let response = self
.head_bucket_request()
@ -211,7 +205,7 @@ impl Store for ObjectStore {
&self,
reader: Reader,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead + Unpin + 'static,
{
@ -224,7 +218,7 @@ impl Store for ObjectStore {
&self,
mut stream: S,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError>
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
{
@ -283,7 +277,7 @@ impl Store for ObjectStore {
let object_id2 = object_id.clone();
let upload_id2 = upload_id.clone();
let handle = actix_rt::spawn(
let handle = crate::sync::spawn(
async move {
let response = this
.create_upload_part_request(
@ -363,7 +357,7 @@ impl Store for ObjectStore {
&self,
bytes: Bytes,
content_type: mime::Mime,
) -> Result<Self::Identifier, StoreError> {
) -> Result<Arc<str>, StoreError> {
let (req, object_id) = self.put_object_request(bytes.len(), content_type).await?;
let response = req.body(bytes).send().await.map_err(ObjectError::from)?;
@ -375,9 +369,9 @@ impl Store for ObjectStore {
Ok(object_id)
}
fn public_url(&self, identifier: &Self::Identifier) -> Option<url::Url> {
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
self.public_endpoint.clone().map(|mut endpoint| {
endpoint.set_path(identifier.as_str());
endpoint.set_path(identifier.as_ref());
endpoint
})
}
@ -385,10 +379,10 @@ impl Store for ObjectStore {
#[tracing::instrument(skip(self))]
async fn to_stream(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
from_start: Option<u64>,
len: Option<u64>,
) -> Result<Self::Stream, StoreError> {
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
let response = self
.get_object_request(identifier, from_start, len)
.send()
@ -409,7 +403,7 @@ impl Store for ObjectStore {
#[tracing::instrument(skip(self, writer))]
async fn read_into<Writer>(
&self,
identifier: &Self::Identifier,
identifier: &Arc<str>,
writer: &mut Writer,
) -> Result<(), std::io::Error>
where
@ -440,7 +434,7 @@ impl Store for ObjectStore {
}
#[tracing::instrument(skip(self))]
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, StoreError> {
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
let response = self
.head_object_request(identifier)
.send()
@ -464,7 +458,7 @@ impl Store for ObjectStore {
}
#[tracing::instrument(skip(self))]
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), StoreError> {
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
let response = self
.delete_object_request(identifier)
.send()
@ -493,7 +487,7 @@ impl ObjectStore {
signature_expiration: u64,
client_timeout: u64,
public_endpoint: Option<Url>,
repo: Repo,
repo: ArcRepo,
) -> Result<ObjectStoreConfig, StoreError> {
let path_gen = init_generator(&repo).await?;
@ -523,7 +517,7 @@ impl ObjectStore {
&self,
length: usize,
content_type: mime::Mime,
) -> Result<(RequestBuilder, ObjectId), StoreError> {
) -> Result<(RequestBuilder, Arc<str>), StoreError> {
let path = self.next_file().await?;
let mut action = self.bucket.put_object(Some(&self.credentials), &path);
@ -535,13 +529,13 @@ impl ObjectStore {
.headers_mut()
.insert("content-length", length.to_string());
Ok((self.build_request(action), ObjectId::from_string(path)))
Ok((self.build_request(action), Arc::from(path)))
}
async fn create_multipart_request(
&self,
content_type: mime::Mime,
) -> Result<(RequestBuilder, ObjectId), StoreError> {
) -> Result<(RequestBuilder, Arc<str>), StoreError> {
let path = self.next_file().await?;
let mut action = self
@ -552,13 +546,13 @@ impl ObjectStore {
.headers_mut()
.insert("content-type", content_type.as_ref());
Ok((self.build_request(action), ObjectId::from_string(path)))
Ok((self.build_request(action), Arc::from(path)))
}
async fn create_upload_part_request(
&self,
buf: BytesStream,
object_id: &ObjectId,
object_id: &Arc<str>,
part_number: u16,
upload_id: &str,
) -> Result<RequestBuilder, ObjectError> {
@ -566,7 +560,7 @@ impl ObjectStore {
let mut action = self.bucket.upload_part(
Some(&self.credentials),
object_id.as_str(),
object_id.as_ref(),
part_number,
upload_id,
);
@ -601,13 +595,13 @@ impl ObjectStore {
async fn send_complete_multipart_request<'a, I: Iterator<Item = &'a str>>(
&'a self,
object_id: &'a ObjectId,
object_id: &'a Arc<str>,
upload_id: &'a str,
etags: I,
) -> Result<Response, reqwest_middleware::Error> {
let mut action = self.bucket.complete_multipart_upload(
Some(&self.credentials),
object_id.as_str(),
object_id.as_ref(),
upload_id,
etags,
);
@ -628,12 +622,12 @@ impl ObjectStore {
fn create_abort_multipart_request(
&self,
object_id: &ObjectId,
object_id: &Arc<str>,
upload_id: &str,
) -> RequestBuilder {
let action = self.bucket.abort_multipart_upload(
Some(&self.credentials),
object_id.as_str(),
object_id.as_ref(),
upload_id,
);
@ -671,13 +665,13 @@ impl ObjectStore {
fn get_object_request(
&self,
identifier: &ObjectId,
identifier: &Arc<str>,
from_start: Option<u64>,
len: Option<u64>,
) -> RequestBuilder {
let action = self
.bucket
.get_object(Some(&self.credentials), identifier.as_str());
.get_object(Some(&self.credentials), identifier.as_ref());
let req = self.build_request(action);
@ -695,18 +689,18 @@ impl ObjectStore {
)
}
fn head_object_request(&self, identifier: &ObjectId) -> RequestBuilder {
fn head_object_request(&self, identifier: &Arc<str>) -> RequestBuilder {
let action = self
.bucket
.head_object(Some(&self.credentials), identifier.as_str());
.head_object(Some(&self.credentials), identifier.as_ref());
self.build_request(action)
}
fn delete_object_request(&self, identifier: &ObjectId) -> RequestBuilder {
fn delete_object_request(&self, identifier: &Arc<str>) -> RequestBuilder {
let action = self
.bucket
.delete_object(Some(&self.credentials), identifier.as_str());
.delete_object(Some(&self.credentials), identifier.as_ref());
self.build_request(action)
}
@ -714,13 +708,9 @@ impl ObjectStore {
async fn next_directory(&self) -> Result<Path, StoreError> {
let path = self.path_gen.next();
match self.repo {
Repo::Sled(ref sled_repo) => {
sled_repo
.set(GENERATOR_KEY, path.to_be_bytes().into())
.await?;
}
}
self.repo
.set(GENERATOR_KEY, path.to_be_bytes().into())
.await?;
Ok(path)
}
@ -733,18 +723,14 @@ impl ObjectStore {
}
}
async fn init_generator(repo: &Repo) -> Result<Generator, StoreError> {
match repo {
Repo::Sled(sled_repo) => {
if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
Ok(Generator::from_existing(
storage_path_generator::Path::from_be_bytes(ivec.to_vec())
.map_err(ObjectError::from)?,
))
} else {
Ok(Generator::new())
}
}
async fn init_generator(repo: &ArcRepo) -> Result<Generator, StoreError> {
if let Some(ivec) = repo.get(GENERATOR_KEY).await? {
Ok(Generator::from_existing(
storage_path_generator::Path::from_be_bytes(ivec.to_vec())
.map_err(ObjectError::from)?,
))
} else {
Ok(Generator::new())
}
}

37
src/store/object_store/object_id.rs
View File

@ -1,37 +0,0 @@
use crate::store::{object_store::ObjectError, Identifier, StoreError};
#[derive(Debug, Clone)]
pub(crate) struct ObjectId(String);
impl Identifier for ObjectId {
fn to_bytes(&self) -> Result<Vec<u8>, StoreError> {
Ok(self.0.as_bytes().to_vec())
}
fn from_bytes(bytes: Vec<u8>) -> Result<Self, StoreError> {
Ok(ObjectId(
String::from_utf8(bytes).map_err(ObjectError::from)?,
))
}
fn from_arc(arc: std::sync::Arc<[u8]>) -> Result<Self, StoreError>
where
Self: Sized,
{
Self::from_bytes(Vec::from(&arc[..]))
}
fn string_repr(&self) -> String {
self.0.clone()
}
}
impl ObjectId {
pub(super) fn from_string(string: String) -> Self {
ObjectId(string)
}
pub(super) fn as_str(&self) -> &str {
&self.0
}
}

27
src/stream.rs
View File

@ -1,5 +1,6 @@
use actix_rt::{task::JoinHandle, time::Sleep};
use actix_web::web::Bytes;
use flume::r#async::RecvStream;
use futures_core::Stream;
use std::{
future::Future,
@ -174,19 +175,25 @@ pin_project_lite::pin_project! {
}
}
enum IterStreamState<I, T> {
enum IterStreamState<I, T>
where
T: 'static,
{
New {
iterator: I,
buffer: usize,
},
Running {
handle: JoinHandle<()>,
receiver: tokio::sync::mpsc::Receiver<T>,
receiver: RecvStream<'static, T>,
},
Pending,
}
pub(crate) struct IterStream<I, T> {
pub(crate) struct IterStream<I, T>
where
T: 'static,
{
state: IterStreamState<I, T>,
}
@ -287,14 +294,13 @@ where
match std::mem::replace(&mut this.state, IterStreamState::Pending) {
IterStreamState::New { iterator, buffer } => {
let (sender, receiver) = tracing::trace_span!(parent: None, "Create channel")
.in_scope(|| tokio::sync::mpsc::channel(buffer));
let (sender, receiver) = crate::sync::channel(buffer);
let mut handle = actix_rt::task::spawn_blocking(move || {
let mut handle = crate::sync::spawn_blocking(move || {
let iterator = iterator.into_iter();
for item in iterator {
if sender.blocking_send(item).is_err() {
if sender.send(item).is_err() {
break;
}
}
@ -304,14 +310,17 @@ where
return Poll::Ready(None);
}
this.state = IterStreamState::Running { handle, receiver };
this.state = IterStreamState::Running {
handle,
receiver: receiver.into_stream(),
};
self.poll_next(cx)
}
IterStreamState::Running {
mut handle,
mut receiver,
} => match Pin::new(&mut receiver).poll_recv(cx) {
} => match Pin::new(&mut receiver).poll_next(cx) {
Poll::Ready(Some(item)) => {
this.state = IterStreamState::Running { handle, receiver };

38
src/sync.rs Normal file
View File

@ -0,0 +1,38 @@
use std::sync::Arc;
use tokio::sync::{Notify, Semaphore};
pub(crate) fn channel<T>(bound: usize) -> (flume::Sender<T>, flume::Receiver<T>) {
tracing::trace_span!(parent: None, "make channel").in_scope(|| flume::bounded(bound))
}
pub(crate) fn notify() -> Arc<Notify> {
Arc::new(bare_notify())
}
pub(crate) fn bare_notify() -> Notify {
tracing::trace_span!(parent: None, "make notifier").in_scope(Notify::new)
}
pub(crate) fn bare_semaphore(permits: usize) -> Semaphore {
tracing::trace_span!(parent: None, "make semaphore").in_scope(|| Semaphore::new(permits))
}
pub(crate) fn spawn<F>(future: F) -> actix_rt::task::JoinHandle<F::Output>
where
F: std::future::Future + 'static,
F::Output: 'static,
{
tracing::trace_span!(parent: None, "spawn task").in_scope(|| actix_rt::spawn(future))
}
pub(crate) fn spawn_blocking<F, Out>(function: F) -> actix_rt::task::JoinHandle<Out>
where
F: FnOnce() -> Out + Send + 'static,
Out: Send + 'static,
{
let outer_span = tracing::Span::current();
tracing::trace_span!(parent: None, "spawn blocking task")
.in_scope(|| actix_rt::task::spawn_blocking(move || outer_span.in_scope(function)))
}

3
src/tmp_file.rs
View File

@ -13,8 +13,7 @@ struct TmpFile(PathBuf);
impl Drop for TmpFile {
fn drop(&mut self) {
tracing::trace_span!(parent: None, "Spawn task")
.in_scope(|| actix_rt::spawn(tokio::fs::remove_file(self.0.clone())));
crate::sync::spawn(tokio::fs::remove_file(self.0.clone()));
}
}