Add console-subscriber (#2003)

* Add console-subscriber

* Add larg event buffer capacity

* Add console port to lemmy prod container

* Expose pict-rs console port

* Update pictrs

.cargo/config

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable"]

Cargo.lock

@@ -690,6 +690,40 @@ dependencies = [
  "xdg",
 ]
 
+[[package]]
+name = "console-api"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "033fddce299c93dd44ae21d5f5a6e749baa5d103784bcdde65701c07272a9fde"
+dependencies = [
+ "prost 0.9.0",
+ "prost-types 0.9.0",
+ "tonic 0.6.2",
+ "tonic-build 0.6.2",
+ "tracing-core",
+]
+
+[[package]]
+name = "console-subscriber"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2380cc150266375aeda8f9aeadc5527395c1a8807ecf9fa97a46d1bb760ec5b"
+dependencies = [
+ "console-api",
+ "futures",
+ "hdrhistogram",
+ "humantime",
+ "serde",
+ "serde_json",
+ "thread_local",
+ "tokio",
+ "tokio-stream",
+ "tonic 0.6.2",
+ "tracing",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -1058,6 +1092,70 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
+[[package]]
+name = "encoding"
+version = "0.2.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
+dependencies = [
+ "encoding-index-japanese",
+ "encoding-index-korean",
+ "encoding-index-simpchinese",
+ "encoding-index-singlebyte",
+ "encoding-index-tradchinese",
+]
+
+[[package]]
+name = "encoding-index-japanese"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-korean"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-simpchinese"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-singlebyte"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-tradchinese"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding_index_tests"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.30"
@@ -1115,6 +1213,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
 
+[[package]]
+name = "fixedbitset"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"
+
 [[package]]
 name = "flate2"
 version = "1.0.22"
@@ -1348,6 +1452,19 @@ dependencies = [
  "ahash",
 ]
 
+[[package]]
+name = "hdrhistogram"
+version = "7.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6490be71f07a5f62b564bc58e36953f675833df11c7e4a0647bee7a07ca1ec5e"
+dependencies = [
+ "base64 0.13.0",
+ "byteorder",
+ "flate2",
+ "nom 7.1.0",
+ "num-traits",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.3"
@@ -1494,6 +1611,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
 [[package]]
 name = "hyper"
 version = "0.14.16"
@@ -1971,6 +2094,7 @@ dependencies = [
  "cargo-husky",
  "chrono",
  "clokwerk",
+ "console-subscriber",
  "diesel",
  "diesel_migrations",
  "doku",
@@ -2018,6 +2142,7 @@ dependencies = [
  "deser-hjson",
  "diesel",
  "doku",
+ "encoding",
  "futures",
  "http",
  "itertools",
@@ -2542,11 +2667,11 @@ dependencies = [
  "futures",
  "http",
  "opentelemetry",
- "prost",
+ "prost 0.8.0",
  "thiserror",
  "tokio",
- "tonic",
+ "tonic 0.5.2",
- "tonic-build",
+ "tonic-build 0.5.2",
 ]
 
 [[package]]
@@ -2646,7 +2771,17 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
 dependencies = [
- "fixedbitset",
+ "fixedbitset 0.2.0",
+ "indexmap",
+]
+
+[[package]]
+name = "petgraph"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f"
+dependencies = [
+ "fixedbitset 0.4.1",
  "indexmap",
 ]
 
@@ -2784,7 +2919,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
 dependencies = [
  "bytes",
- "prost-derive",
+ "prost-derive 0.8.0",
+]
+
+[[package]]
+name = "prost"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
+dependencies = [
+ "bytes",
+ "prost-derive 0.9.0",
 ]
 
 [[package]]
@@ -2798,9 +2943,29 @@ dependencies = [
  "itertools",
  "log",
  "multimap",
- "petgraph",
+ "petgraph 0.5.1",
- "prost",
+ "prost 0.8.0",
- "prost-types",
+ "prost-types 0.8.0",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
+dependencies = [
+ "bytes",
+ "heck",
+ "itertools",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph 0.6.0",
+ "prost 0.9.0",
+ "prost-types 0.9.0",
+ "regex",
  "tempfile",
  "which",
 ]
@@ -2818,6 +2983,19 @@ dependencies = [
  "syn 1.0.82",
 ]
 
+[[package]]
+name = "prost-derive"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2 1.0.33",
+ "quote 1.0.10",
+ "syn 1.0.82",
+]
+
 [[package]]
 name = "prost-types"
 version = "0.8.0"
@@ -2825,7 +3003,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
 dependencies = [
  "bytes",
- "prost",
+ "prost 0.8.0",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
+dependencies = [
+ "bytes",
+ "prost 0.9.0",
 ]
 
 [[package]]
@@ -3693,11 +3881,10 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144"
+checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838"
 dependencies = [
- "autocfg",
  "bytes",
  "libc",
  "memchr",
@@ -3708,6 +3895,7 @@ dependencies = [
  "pin-project-lite",
  "signal-hook-registry",
  "tokio-macros",
+ "tracing",
  "winapi",
 ]
 
@@ -3723,9 +3911,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "1.6.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e"
+checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
 dependencies = [
  "proc-macro2 1.0.33",
  "quote 1.0.10",
@@ -3806,8 +3994,39 @@ dependencies = [
  "hyper-timeout",
  "percent-encoding",
  "pin-project",
- "prost",
+ "prost 0.8.0",
- "prost-derive",
+ "prost-derive 0.8.0",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tonic"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "base64 0.13.0",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost 0.9.0",
+ "prost-derive 0.9.0",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -3825,7 +4044,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08"
 dependencies = [
  "proc-macro2 1.0.33",
- "prost-build",
+ "prost-build 0.8.0",
+ "quote 1.0.10",
+ "syn 1.0.82",
+]
+
+[[package]]
+name = "tonic-build"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"
+dependencies = [
+ "proc-macro2 1.0.33",
+ "prost-build 0.9.0",
  "quote 1.0.10",
  "syn 1.0.82",
 ]

Cargo.toml

@@ -54,11 +54,12 @@ tracing-actix-web = { version = "0.5.0-beta.8", default-features = false }
 tracing-error = "0.2.0"
 tracing-log = "0.1.2"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
+console-subscriber = "0.1.0"
 strum = "0.23.0"
 url = { version = "2.2.2", features = ["serde"] }
 openssl = "0.10.38"
 http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] }
-tokio = { version = "1.14.0", features = ["sync"] }
+tokio = { version = "1.14.0", features = ["sync", "tracing"] }
 anyhow = "1.0.51"
 reqwest = { version = "0.11.7", features = ["json"] }
 reqwest-middleware = "0.1.3"

crates/api/Cargo.toml

@@ -39,7 +39,7 @@ http = "0.2.5"
 http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] }
 base64 = "0.13.0"
 tokio = "1.14.0"
-futures = "0.3.18"
+futures = "0.3.17"
 itertools = "0.10.3"
 uuid = { version = "0.8.2", features = ["serde", "v4"] }
 sha2 = "0.10.0"

crates/api_crud/Cargo.toml

@@ -35,7 +35,7 @@ http = "0.2.5"
 http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] }
 base64 = "0.13.0"
 tokio = "1.14.0"
-futures = "0.3.18"
+futures = "0.3.17"
 itertools = "0.10.3"
 uuid = { version = "0.8.2", features = ["serde", "v4"] }
 sha2 = "0.10.0"

crates/apub/Cargo.toml

@@ -39,7 +39,7 @@ percent-encoding = "2.1.0"
 http = "0.2.5"
 http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["server", "sha-2"] }
 tokio = "1.14.0"
-futures = "0.3.18"
+futures = "0.3.17"
 itertools = "0.10.3"
 uuid = { version = "0.8.2", features = ["serde", "v4"] }
 sha2 = "0.10.0"

crates/apub_lib/src/activity_queue.rs

@@ -82,9 +82,10 @@ async fn do_send(task: SendActivityTask, client: &ClientWithMiddleware) -> Resul
       if !o.status().is_success() {
         let status = o.status();
         let text = o.text().await?;
+
         warn!(
           "Send {} to {} failed with status {}: {}",
-          task.activity_id, task.inbox, status, text
+          task.activity_id, task.inbox, status, text,
         );
       }
     }

crates/routes/Cargo.toml

@@ -26,7 +26,7 @@ actix-http = "3.0.0-beta.15"
 sha2 = "0.10.0"
 anyhow = "1.0.51"
 chrono = { version = "0.4.19", features = ["serde"] }
-futures = "0.3.18"
+futures = "0.3.17"
 reqwest = { version = "0.11.7", features = ["stream"] }
 reqwest-middleware = "0.1.3"
 rss = "2.0.0"

crates/utils/Cargo.toml

@@ -36,7 +36,7 @@ reqwest-middleware = "0.1.3"
 tokio = { version = "1.14.0", features = ["sync"] }
 strum = "0.23.0"
 strum_macros = "0.23.1"
-futures = "0.3.18"
+futures = "0.3.17"
 diesel = "1.4.8"
 http = "0.2.5"
 deser-hjson = "1.0.2"

docker/dev/Dockerfile

@@ -7,6 +7,7 @@ WORKDIR /app
 
 # Cargo chef plan
 FROM chef as planner
+ENV RUSTFLAGS="--cfg tokio_unstable"
 
 # Copy dirs
 COPY . .
@@ -16,6 +17,7 @@ RUN cargo chef prepare --recipe-path recipe.json
 FROM chef as builder
 ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
 ARG RUSTRELEASEDIR="debug"
+ENV RUSTFLAGS="--cfg tokio_unstable"
 
 COPY --from=planner /app/recipe.json ./recipe.json
 RUN cargo chef cook --recipe-path recipe.json --target ${CARGO_BUILD_TARGET}

docker/dev/docker-compose.yml

@@ -16,6 +16,7 @@ services:
     image: lemmy-dev:latest
     ports:
       - "8536:8536"
+      - "6669:6669"
     restart: always
     environment:
       - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
@@ -50,10 +51,12 @@ services:
     restart: always
 
   pictrs:
-    image: asonix/pictrs:0.3.0-beta.11
+    image: asonix/pictrs:0.3.0-beta.12-r1
     user: 991:991
     environment:
       - PICTRS_OPENTELEMETRY_URL=http://otel:4137
+    ports:
+      - "6670:6669"
     volumes:
       - ./volumes/pictrs:/mnt
     restart: always

docker/prod/docker-compose.yml

@@ -15,6 +15,7 @@ services:
     image: dessalines/lemmy:0.14.3
     ports:
       - "127.0.0.1:8536:8536"
+      - "127.0.0.1:6669:6669"
     restart: always
     environment:
       - RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
@@ -37,9 +38,10 @@ services:
       - lemmy
 
   pictrs:
-    image: asonix/pictrs:0.3.0-beta.11
+    image: asonix/pictrs:0.3.0-beta.12-r1
     ports: 
       - "127.0.0.1:8537:8080"
+      - "127.0.0.1:6670:6669"
     user: 991:991
     volumes:
       - ./volumes/pictrs:/mnt

src/lib.rs

@@ -4,6 +4,7 @@ pub mod code_migrations;
 pub mod root_span_builder;
 pub mod scheduled_tasks;
 
+use console_subscriber::ConsoleLayer;
 use lemmy_utils::LemmyError;
 use opentelemetry::{
   sdk::{propagation::TraceContextPropagator, Resource},
@@ -13,20 +14,32 @@ use opentelemetry_otlp::WithExportConfig;
 use tracing::subscriber::set_global_default;
 use tracing_error::ErrorLayer;
 use tracing_log::LogTracer;
-use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
+use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry};
 
 pub fn init_tracing(opentelemetry_url: Option<&str>) -> Result<(), LemmyError> {
   LogTracer::init()?;
 
   opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
 
-  let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
+  let log_description = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into());
-  let format_layer = tracing_subscriber::fmt::layer();
+
+  let targets = log_description
+    .trim()
+    .trim_matches('"')
+    .parse::<Targets>()?;
+
+  let format_layer = tracing_subscriber::fmt::layer().with_filter(targets.clone());
+
+  let console_layer = ConsoleLayer::builder()
+    .with_default_env()
+    .server_addr(([0, 0, 0, 0], 6669))
+    .event_buffer_capacity(1024 * 1024)
+    .spawn();
 
   let subscriber = Registry::default()
-    .with(env_filter)
     .with(format_layer)
-    .with(ErrorLayer::default());
+    .with(ErrorLayer::default())
+    .with(console_layer);
 
   if let Some(url) = opentelemetry_url {
     let tracer = opentelemetry_otlp::new_pipeline()
@@ -42,7 +55,9 @@ pub fn init_tracing(opentelemetry_url: Option<&str>) -> Result<(), LemmyError> {
       )
       .install_batch(opentelemetry::runtime::Tokio)?;
 
-    let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
+    let otel_layer = tracing_opentelemetry::layer()
+      .with_tracer(tracer)
+      .with_filter(targets);
 
     let subscriber = subscriber.with(otel_layer);