From 53a2b6d0130b2eb53fa33488e25e772733bd4e74 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 8 Oct 2021 10:36:53 -0400 Subject: [PATCH] Trying a background_jobs fix. #1820 (#1822) --- Cargo.lock | 183 +++++++++++++------------- crates/api/Cargo.toml | 2 +- crates/api_crud/Cargo.toml | 2 +- crates/apub/Cargo.toml | 1 + crates/apub_lib/Cargo.toml | 2 +- crates/apub_lib/src/activity_queue.rs | 3 +- crates/websocket/Cargo.toml | 2 +- src/scheduled_tasks.rs | 3 +- 8 files changed, 100 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75a1a1c70..6c5adba31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,8 +103,8 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837" dependencies = [ - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -256,8 +256,8 @@ checksum = "4a11fd6f322120a74b23327e778ef0a4950b1f44a2b76468a69316a150f5c6dd" dependencies = [ "actix-router", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -267,8 +267,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d44b8fee1ced9671ba043476deddef739dd0959bf77030b26b738cc591737a7" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -331,8 +331,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -398,9 +398,9 @@ dependencies = [ [[package]] name = "background-jobs" -version = "0.10.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0042f85da71ac239f62131239bcfa83686e52a3eba4738056a5c09163dd21d4b" +checksum = "1ce8953ce13a4bc359900fca168b64e47d5f0af1c02f2d45527221babbb73a27" dependencies = [ "background-jobs-actix", "background-jobs-core", @@ -408,9 +408,9 @@ dependencies = [ [[package]] name = "background-jobs-actix" -version = "0.10.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "827b3bfdd534ab88fdd3fd1598496fe4de09f08246761b5d2b677f6bd9a60b5b" +checksum = "709bf5f9fc7ac9d99166a519170bbc0a8f72bb37673c15172c09b4a12e339684" dependencies = [ "actix-rt", "anyhow", @@ -418,33 +418,31 @@ dependencies = [ "async-trait", "background-jobs-core", "chrono", + "log", "num_cpus", "serde", "serde_json", "thiserror", "tokio", - "tracing", - "tracing-futures", "uuid", ] [[package]] name = "background-jobs-core" -version = "0.10.0" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afbf8f0b4cf2ad0e610e05245e93fd3c94aeacbc35264543c4280ecf81bbce63" +checksum = "19183dba9542792f7abd93c9078a93db49eb7b035d733bc135c2fac9ec822926" dependencies = [ "actix-rt", "anyhow", "async-mutex", "async-trait", "chrono", + "log", "serde", "serde_json", "thiserror", "tokio", - "tracing", - "tracing-futures", "uuid", ] @@ -612,9 +610,9 @@ checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" [[package]] name = "cc" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" +checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" [[package]] name = "cfg-if" @@ -817,9 +815,9 @@ dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "strsim 0.9.3", - "syn 1.0.77", + "syn 1.0.80", ] [[package]] @@ -831,9 +829,9 @@ dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "strsim 0.10.0", - "syn 1.0.77", + "syn 1.0.80", ] [[package]] @@ -843,8 +841,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ "darling_core 0.10.2", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -854,8 +852,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc" dependencies = [ "darling_core 0.13.0", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -877,8 +875,8 @@ dependencies = [ "darling 0.10.2", "derive_builder_core", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -889,8 +887,8 @@ checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef" dependencies = [ "darling 0.10.2", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -901,9 +899,9 @@ checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df" dependencies = [ "convert_case", "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "rustc_version 0.3.3", - "syn 1.0.77", + "syn 1.0.80", ] [[package]] @@ -949,8 +947,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -1194,8 +1192,8 @@ dependencies = [ "autocfg 1.0.1", "proc-macro-hack", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -1290,9 +1288,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472" +checksum = "6c06815895acec637cd6ed6e9662c935b866d20a106f8361892893a7d9234964" dependencies = [ "bytes", "fnv", @@ -1375,8 +1373,8 @@ dependencies = [ "mac", "markup5ever 0.10.1", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -1746,6 +1744,7 @@ dependencies = [ "anyhow", "async-trait", "awc", + "background-jobs", "bcrypt", "chrono", "diesel", @@ -1807,8 +1806,8 @@ name = "lemmy_apub_lib_derive" version = "0.13.0" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", "trybuild", ] @@ -2196,8 +2195,8 @@ checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" dependencies = [ "migrations_internals", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -2482,8 +2481,8 @@ dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -2589,8 +2588,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -2695,9 +2694,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" dependencies = [ "proc-macro2 1.0.29", ] @@ -3228,8 +3227,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -3275,8 +3274,8 @@ checksum = "98c1fcca18d55d1763e1c16873c4bde0ac3ef75179a28c7b372917e0494625be" dependencies = [ "darling 0.13.0", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -3297,8 +3296,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -3411,8 +3410,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -3467,10 +3466,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "serde", "serde_derive", - "syn 1.0.77", + "syn 1.0.80", ] [[package]] @@ -3481,12 +3480,12 @@ checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "serde", "serde_derive", "serde_json", "sha1", - "syn 1.0.77", + "syn 1.0.80", ] [[package]] @@ -3532,7 +3531,7 @@ dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "string_cache_shared", ] @@ -3545,7 +3544,7 @@ dependencies = [ "phf_generator 0.8.0", "phf_shared 0.8.0", "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", ] [[package]] @@ -3580,8 +3579,8 @@ checksum = "d06aaeeee809dbc59eb4556183dd927df67db1540de5be8d3ec0b6636358a5ec" dependencies = [ "heck", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -3608,12 +3607,12 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.77" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5239bc68e0fef57495900cfea4e8dc75596d9a319d7e16b1e0a440d24e6fe0a0" +checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "unicode-xid 0.2.2", ] @@ -3673,8 +3672,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] @@ -3741,9 +3740,9 @@ checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" dependencies = [ "proc-macro-hack", "proc-macro2 1.0.29", - "quote 1.0.9", + "quote 1.0.10", "standback", - "syn 1.0.77", + "syn 1.0.80", ] [[package]] @@ -3831,9 +3830,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f96e095c0c82419687c20ddf5cb3eadb61f4e1405923c9dc8e53a1adacbda8" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ "cfg-if", "pin-project-lite", @@ -3843,20 +3842,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.16" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98863d0dd09fa59a1b79c6750ad80dbda6b75f4e71c437a6a1a8cb91a8bcbd77" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", ] [[package]] name = "tracing-core" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46125608c26121c81b0c6d693eab5a420e416da7e43c426d2e8f7df8da8a3acf" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" dependencies = [ "lazy_static", ] @@ -3883,9 +3882,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd0568dbfe3baf7048b7908d2b32bca0d81cd56bec6d2a8f894b01d74f86be3" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" dependencies = [ "sharded-slab", "thread_local", @@ -3900,9 +3899,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "trybuild" -version = "1.0.45" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bdaf2a1d317f3d58b44b31c7f6436b9b9acafe7bddfeace50897c2b804d7792" +checksum = "4e6088028c71b42d26126c208a072670bcb964cfe73904c331b05e9c26080470" dependencies = [ "dissimilar", "glob", @@ -4077,8 +4076,8 @@ dependencies = [ "lazy_static", "log", "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", "wasm-bindgen-shared", ] @@ -4100,7 +4099,7 @@ version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ - "quote 1.0.9", + "quote 1.0.10", "wasm-bindgen-macro-support", ] @@ -4111,8 +4110,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ "proc-macro2 1.0.29", - "quote 1.0.9", - "syn 1.0.77", + "quote 1.0.10", + "syn 1.0.80", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index babb90151..4448a3423 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -48,5 +48,5 @@ async-trait = "0.1.51" captcha = "0.0.8" anyhow = "1.0.44" thiserror = "1.0.29" -background-jobs = "0.10.0" +background-jobs = "0.9.0" reqwest = { version = "0.11.4", features = ["json"] } diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index a5c8e6c52..7666e220c 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -43,6 +43,6 @@ sha2 = "0.9.8" async-trait = "0.1.51" anyhow = "1.0.44" thiserror = "1.0.29" -background-jobs = "0.10.0" +background-jobs = "0.9.0" reqwest = { version = "0.11.4", features = ["json"] } webmention = "0.4.0" diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index 2d49d53ce..6e700f88a 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -46,5 +46,6 @@ sha2 = "0.9.8" async-trait = "0.1.51" anyhow = "1.0.44" thiserror = "1.0.29" +background-jobs = "0.9.0" reqwest = { version = "0.11.4", features = ["json"] } diff --git a/crates/apub_lib/Cargo.toml b/crates/apub_lib/Cargo.toml index ba21ce0a6..0e56edf00 100644 --- a/crates/apub_lib/Cargo.toml +++ b/crates/apub_lib/Cargo.toml @@ -24,4 +24,4 @@ sha2 = "0.9.8" actix-web = { version = "4.0.0-beta.9", default-features = false } http-signature-normalization-actix = { version = "0.5.0-beta.10", default-features = false, features = ["server", "sha-2"] } http-signature-normalization-reqwest = { version = "0.2.0", default-features = false, features = ["sha-2"] } -background-jobs = "0.10.0" +background-jobs = "0.9.0" diff --git a/crates/apub_lib/src/activity_queue.rs b/crates/apub_lib/src/activity_queue.rs index ec1ede46b..7efa54ef6 100644 --- a/crates/apub_lib/src/activity_queue.rs +++ b/crates/apub_lib/src/activity_queue.rs @@ -90,13 +90,14 @@ async fn do_send(task: SendActivityTask, client: &Client) -> Result<(), Error> { pub fn create_activity_queue() -> QueueHandle { // Start the application server. This guards access to to the jobs store let queue_handle = create_server(Storage::new()); + let arbiter = actix_web::rt::Arbiter::new(); // Configure and start our workers WorkerConfig::new(|| MyState { client: Client::default(), }) .register::() - .start(queue_handle.clone()); + .start_in_arbiter(&arbiter, queue_handle.clone()); queue_handle } diff --git a/crates/websocket/Cargo.toml b/crates/websocket/Cargo.toml index 7a22e207e..7a3fbf965 100644 --- a/crates/websocket/Cargo.toml +++ b/crates/websocket/Cargo.toml @@ -25,7 +25,7 @@ serde_json = { version = "1.0.68", features = ["preserve_order"] } actix = "0.12.0" anyhow = "1.0.44" diesel = "1.4.8" -background-jobs = "0.10.0" +background-jobs = "0.9.0" tokio = "1.12.0" strum = "0.21.0" strum_macros = "0.21.1" diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index b4001f80d..5c406ff6d 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -15,7 +15,8 @@ pub fn setup(pool: DbPool) { active_counts(&conn); // On startup, reindex the tables non-concurrently - reindex_aggregates_tables(&conn, false); + // TODO remove this for now, since it slows down startup a lot on lemmy.ml + reindex_aggregates_tables(&conn, true); scheduler.every(1.hour()).run(move || { active_counts(&conn); reindex_aggregates_tables(&conn, true);