From 418db7831ff1d3035dfcfe582dd3bb76f52e58d6 Mon Sep 17 00:00:00 2001
From: Felix Ableitner <me@nutomic.com>
Date: Fri, 28 Oct 2022 14:05:14 +0200
Subject: [PATCH] Log crawl errors

---
 Cargo.lock | 226 ++++++++++++++++++++++-------------------------------
 src/lib.rs |   2 +
 2 files changed, 96 insertions(+), 132 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e606f7b..4471546 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -31,7 +31,7 @@ dependencies = [
  "actix-tls",
  "actix-utils",
  "ahash",
- "base64",
+ "base64 0.13.0",
  "bitflags",
  "bytes",
  "bytestring",
@@ -270,9 +270,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "background-jobs"
-version = "0.12.0"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c301b3ccb60fa2aadb4da111e27b257ddc4f0509848c43c85764dc14f7a49a3"
+checksum = "77f4508c6c5b5cfc6c18d43d0ba6ecda339710206854da9e1c9ac9dfb7e3eb6f"
 dependencies = [
  "background-jobs-actix",
  "background-jobs-core",
@@ -280,15 +280,16 @@ dependencies = [
 
 [[package]]
 name = "background-jobs-actix"
-version = "0.12.0"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65a0dcbc94e0644fcfbaddc19705fae2d6947426682aeab7582d772f4d80b8df"
+checksum = "5dabf6a2204fe034db7910a38f8e2d183fe24eb92abd4c0aaca59f8cacf4e48b"
 dependencies = [
  "actix-rt",
  "anyhow",
  "async-mutex",
  "async-trait",
  "background-jobs-core",
+ "chrono",
  "num_cpus",
  "serde",
  "serde_json",
@@ -301,23 +302,29 @@ dependencies = [
 
 [[package]]
 name = "background-jobs-core"
-version = "0.12.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82fe510c99cea8b98c438e81f43c666e9b4cd3630a42a1684ed1d01db4271cfb"
+checksum = "174d36b170699ecc13b7513bda9eff6f12cc889eae5d16b792daa3f7b21be452"
 dependencies = [
  "actix-rt",
  "anyhow",
  "async-mutex",
  "async-trait",
+ "chrono",
  "serde",
  "serde_json",
  "thiserror",
- "time 0.3.9",
  "tracing",
  "tracing-futures",
  "uuid",
 ]
 
+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.0"
@@ -326,14 +333,13 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
 [[package]]
 name = "bcrypt"
-version = "0.12.1"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fe4fef31efb0f76133ae8e3576a88e58edb7cfc5584c81c758c349ba46b43fc"
+checksum = "f691e63585950d8c1c43644d11bab9073e40f5060dd2822734ae7c3dc69a3a80"
 dependencies = [
- "base64",
+ "base64 0.13.0",
  "blowfish",
  "getrandom 0.2.6",
- "zeroize",
 ]
 
 [[package]]
@@ -374,12 +380,13 @@ dependencies = [
 
 [[package]]
 name = "blowfish"
-version = "0.9.1"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7"
+checksum = "fe3ff3fc1de48c1ac2e3341c4df38b0d1bfb8fdf04632a187c8b75aaa319a7ab"
 dependencies = [
  "byteorder",
  "cipher",
+ "opaque-debug 0.3.0",
 ]
 
 [[package]]
@@ -443,12 +450,11 @@ dependencies = [
 
 [[package]]
 name = "cipher"
-version = "0.4.3"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e"
+checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7"
 dependencies = [
- "crypto-common",
- "inout",
+ "generic-array 0.14.5",
 ]
 
 [[package]]
@@ -693,9 +699,9 @@ dependencies = [
 
 [[package]]
 name = "doku"
-version = "0.11.0"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7669ad18bd53786107832ce23c4cd1d859d3fe2ed6393ebcc0def93f15e5fa3"
+checksum = "fae0158211454f5969d619e425acf2e046001683dfac3a23fa5bf6c6a8464499"
 dependencies = [
  "doku-derive",
  "serde",
@@ -703,9 +709,9 @@ dependencies = [
 
 [[package]]
 name = "doku-derive"
-version = "0.11.0"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "603fe9f91b4d0e11036df029aeaeffa90b8f97e700104d5d24abb053bf9ba858"
+checksum = "366b1ae88672638e08f97cc9037ce4e5dca6c2b37699a50c72b846a4c654d4bb"
 dependencies = [
  "darling",
  "proc-macro2 1.0.37",
@@ -725,7 +731,7 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6690291166824e467790ac08ba42f241791567e8337bbf00c5a6e87889629f98"
 dependencies = [
- "base64",
+ "base64 0.13.0",
 ]
 
 [[package]]
@@ -1039,12 +1045,6 @@ dependencies = [
  "unicode-segmentation",
 ]
 
-[[package]]
-name = "heck"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
-
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -1067,9 +1067,9 @@ dependencies = [
 
 [[package]]
 name = "html2text"
-version = "0.3.1"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de2444cdc9d7e23756d000d9f49513bed8bb4e3743072fb45c63e6493e277bed"
+checksum = "a26379dcb715e237b96102a12b505c553e2bffa74bae2e54658748d298660ef1"
 dependencies = [
  "html5ever",
  "markup5ever_rcdom",
@@ -1112,6 +1112,16 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "http-signature-normalization"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07499ccdca5530f8dca2c295aa4554afefa617879d381466016ba12bc60a09e9"
+dependencies = [
+ "chrono",
+ "thiserror",
+]
+
 [[package]]
 name = "http-signature-normalization"
 version = "0.6.0"
@@ -1130,9 +1140,9 @@ dependencies = [
  "actix-http",
  "actix-rt",
  "actix-web",
- "base64",
+ "base64 0.13.0",
  "futures-util",
- "http-signature-normalization",
+ "http-signature-normalization 0.6.0",
  "sha2",
  "thiserror",
  "tokio",
@@ -1143,14 +1153,15 @@ dependencies = [
 
 [[package]]
 name = "http-signature-normalization-reqwest"
-version = "0.5.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a15d4498e747f1cb1b0114ad651619932eac790815d2dcef0fa89bd016fd991"
+checksum = "2688d50434bf8a95fd848e54f63debd80e7711e4fd164ede3456ff74ec655b76"
 dependencies = [
- "base64",
+ "base64 0.13.0",
  "bytes",
+ "chrono",
  "http",
- "http-signature-normalization",
+ "http-signature-normalization 0.5.4",
  "reqwest",
  "reqwest-middleware",
  "sha2",
@@ -1247,15 +1258,6 @@ dependencies = [
  "hashbrown",
 ]
 
-[[package]]
-name = "inout"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
-dependencies = [
- "generic-array 0.14.5",
-]
-
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -1297,11 +1299,11 @@ dependencies = [
 
 [[package]]
 name = "jsonwebtoken"
-version = "8.1.0"
+version = "7.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc9051c17f81bae79440afa041b3a278e1de71bfb96d32454b477fd4703ccb6f"
+checksum = "afabcc15e437a6484fc4f12d0fd63068fe457bf93f1c148d3d9649c60b103f32"
 dependencies = [
- "base64",
+ "base64 0.12.3",
  "pem",
  "ring",
  "serde",
@@ -1343,9 +1345,9 @@ dependencies = [
 
 [[package]]
 name = "lemmy_api_common"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63db6b4284eb8bc6cf4153cbd45407ae8f53a57f8252988be25f767b4b3b6f6b"
+checksum = "3dba349b07c04457f90b94659b4cce3a1d4e6b36ca14f6e6d9bdfa85579401c0"
 dependencies = [
  "actix-web",
  "chrono",
@@ -1356,7 +1358,6 @@ dependencies = [
  "lemmy_db_views_actor",
  "lemmy_db_views_moderator",
  "lemmy_utils",
- "rosetta-i18n",
  "serde",
  "serde_json",
  "tracing",
@@ -1365,15 +1366,15 @@ dependencies = [
 
 [[package]]
 name = "lemmy_apub_lib"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29e69734ecaa87bfb7b17bf3df6254afc0ce72dfc2a384d6dee9ee83a5fe9e6a"
+checksum = "6f469cd602f3a710f5623d21b2aa73d02403e89b1ec774d9f069b8766bd69b3c"
 dependencies = [
  "actix-web",
  "anyhow",
  "async-trait",
  "background-jobs",
- "base64",
+ "base64 0.13.0",
  "chrono",
  "diesel",
  "http",
@@ -1394,9 +1395,9 @@ dependencies = [
 
 [[package]]
 name = "lemmy_apub_lib_derive"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0416cc1e98209e30fedf6cb2ea2e4ebf54703d00b29e3b7b56a7ab9a893ea8ef"
+checksum = "f83750d52547d34feae17c8bd5c389e6d88b0dcd7be353b09354df810e75a6a2"
 dependencies = [
  "proc-macro2 1.0.37",
  "quote 1.0.18",
@@ -1405,9 +1406,9 @@ dependencies = [
 
 [[package]]
 name = "lemmy_db_schema"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e58cef7b9da19f66acdad8c2c12527a2a9c2c36def2f48f9fefa7c551e3bdb43"
+checksum = "1faad21c61a12fd01755ab1d47737224d44321d3e2204c9a65c13b1b95b26e46"
 dependencies = [
  "bcrypt",
  "chrono",
@@ -1428,9 +1429,9 @@ dependencies = [
 
 [[package]]
 name = "lemmy_db_views"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4856598867bb58f9a535479499e91f75c8ac651d5d59ac3c4441712fbd39abad"
+checksum = "ec114f99ab401f89f6a2a57268137973fa16088ad6cd9de01a8834429166b4e6"
 dependencies = [
  "diesel",
  "lemmy_db_schema",
@@ -1441,9 +1442,9 @@ dependencies = [
 
 [[package]]
 name = "lemmy_db_views_actor"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e09f108435b44eddf0302f5914265d23444528a5557d1aa567be5ee6ee72e01"
+checksum = "97484d21ed465efe32122cb24072124259c45c857e7aef44dbd4d055fc8838b9"
 dependencies = [
  "diesel",
  "lemmy_db_schema",
@@ -1452,9 +1453,9 @@ dependencies = [
 
 [[package]]
 name = "lemmy_db_views_moderator"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed3538a77cd279876ccc149742ccdff26e67a1fe597d3bbfda8b5acb3a6dd808"
+checksum = "0a9e8befcbfdc417364e6102619e8aa49c1dcb9dc6d81c891f33e0eb9d6455cb"
 dependencies = [
  "diesel",
  "lemmy_db_schema",
@@ -1463,9 +1464,9 @@ dependencies = [
 
 [[package]]
 name = "lemmy_utils"
-version = "0.16.3"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ad89224b58f654b9db79949021fef0bcc24e17131363fa0fd74e3bd4bf8031e"
+checksum = "af06fdef998f5c52e0acf9a0b83b003cdaecafbc40205a1e2c9d7f301814a0f7"
 dependencies = [
  "actix-rt",
  "actix-web",
@@ -1484,14 +1485,11 @@ dependencies = [
  "lettre",
  "once_cell",
  "openssl",
- "parking_lot 0.12.0",
  "percent-encoding",
  "rand 0.8.5",
  "regex",
  "reqwest",
  "reqwest-middleware",
- "rosetta-build",
- "rosetta-i18n",
  "serde",
  "serde_json",
  "smart-default",
@@ -1512,7 +1510,7 @@ version = "0.10.0-rc.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2f6c70001f7ee6c93b6687a06607c7a38f9a7ae460139a496c23da21e95bc289"
 dependencies = [
- "base64",
+ "base64 0.13.0",
  "email-encoding",
  "fastrand",
  "futures-util",
@@ -1737,9 +1735,9 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.3"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
+checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
 dependencies = [
  "autocfg",
  "num-integer",
@@ -1796,6 +1794,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
 [[package]]
 name = "openssl"
 version = "0.10.40"
@@ -1897,11 +1901,13 @@ checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
 
 [[package]]
 name = "pem"
-version = "1.0.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9a3b09a20e374558580a4914d3b7d89bd61b954a5a5e1dcbea98753addb1947"
+checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb"
 dependencies = [
- "base64",
+ "base64 0.13.0",
+ "once_cell",
+ "regex",
 ]
 
 [[package]]
@@ -2111,15 +2117,6 @@ dependencies = [
  "unicode-xid 0.2.3",
 ]
 
-[[package]]
-name = "quickcheck"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
-dependencies = [
- "rand 0.8.5",
-]
-
 [[package]]
 name = "quote"
 version = "0.6.13"
@@ -2288,7 +2285,7 @@ version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb"
 dependencies = [
- "base64",
+ "base64 0.13.0",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -2355,26 +2352,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "rosetta-build"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f697b8b3f19bee20f30dc87213d05ce091c43bc733ab1bfc98b0e5cdd9943f3"
-dependencies = [
- "convert_case",
- "lazy_static",
- "proc-macro2 1.0.37",
- "quote 1.0.18",
- "regex",
- "tinyjson",
-]
-
-[[package]]
-name = "rosetta-i18n"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5298de832602aecc9458398f435d9bff0be57da7aac11221b6ff3d4ef9503de"
-
 [[package]]
 name = "rustc_version"
 version = "0.4.0"
@@ -2402,7 +2379,7 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360"
 dependencies = [
- "base64",
+ "base64 0.13.0",
 ]
 
 [[package]]
@@ -2534,7 +2511,7 @@ dependencies = [
  "block-buffer 0.7.3",
  "digest 0.8.1",
  "fake-simd",
- "opaque-debug",
+ "opaque-debug 0.2.3",
 ]
 
 [[package]]
@@ -2585,14 +2562,13 @@ dependencies = [
 
 [[package]]
 name = "simple_asn1"
-version = "0.6.1"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a762b1c38b9b990c694b9c2f8abe3372ce6a9ceaae6bca39cfc46e054f45745"
+checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b"
 dependencies = [
+ "chrono",
  "num-bigint",
  "num-traits",
- "thiserror",
- "time 0.3.9",
 ]
 
 [[package]]
@@ -2708,7 +2684,7 @@ version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
 dependencies = [
- "heck 0.3.3",
+ "heck",
  "proc-macro-error",
  "proc-macro2 1.0.37",
  "quote 1.0.18",
@@ -2717,17 +2693,17 @@ dependencies = [
 
 [[package]]
 name = "strum"
-version = "0.24.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e96acfc1b70604b8b2f1ffa4c57e59176c7dbb05d556c71ecd2f5498a1dee7f8"
+checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"
 
 [[package]]
 name = "strum_macros"
-version = "0.24.0"
+version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef"
+checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"
 dependencies = [
- "heck 0.4.0",
+ "heck",
  "proc-macro2 1.0.37",
  "quote 1.0.18",
  "rustversion",
@@ -2857,8 +2833,6 @@ dependencies = [
  "itoa",
  "libc",
  "num_threads",
- "quickcheck",
- "serde",
  "time-macros",
 ]
 
@@ -2868,12 +2842,6 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
 
-[[package]]
-name = "tinyjson"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a8304da9f9370f6a6f9020b7903b044aa9ce3470f300a1fba5bc77c78145a16"
-
 [[package]]
 name = "tinyvec"
 version = "1.6.0"
@@ -3417,9 +3385,3 @@ dependencies = [
  "markup5ever",
  "time 0.1.44",
 ]
-
-[[package]]
-name = "zeroize"
-version = "1.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07"
diff --git a/src/lib.rs b/src/lib.rs
index 6a1d311..eaaae07 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,6 +10,7 @@ use semver::Version;
 use std::collections::HashSet;
 use std::sync::Arc;
 use std::time::Duration;
+use log::warn;
 use tokio::sync::Mutex;
 
 pub mod crawl;
@@ -46,6 +47,7 @@ pub async fn start_crawl(
         .await
         .into_iter()
         .flatten()
+        .inspect(|r| if let Err(e) = r { warn!("{}", e)})
         .filter_map(|r| r.ok())
         .collect();