From 957638359768a9fc33b2ed71df0d3b7cb8832142 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Thu, 25 Apr 2024 09:07:59 -0700 Subject: [PATCH] add optional support for tokio-console This turned out to be quite hairy, mostly because we need to apply the config's log level filter to the actual logs (stdout and, optionally sentry), but do not want to filter out the tokio tracing events needed by the console_subscriber. I hit several edge cases in tracing getting this to work, and we now depend on a git version of tracing with a backported patch :( --- Cargo.lock | 420 +++++++++++++++++++--- Cargo.toml | 18 + docs/development.md | 18 + src/database/mod.rs | 12 +- src/main.rs | 103 ++++-- src/service/admin/debug/debug_commands.rs | 4 +- src/service/globals/mod.rs | 8 +- src/service/mod.rs | 8 +- 8 files changed, 487 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e44505a8..863b666c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +[[package]] +name = "anyhow" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" + [[package]] name = "arc-swap" version = "1.7.1" @@ -137,6 +143,34 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.7.5" @@ -144,13 +178,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.3", "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-util", "itoa", "matchit", @@ -169,6 +203,23 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.3" @@ -178,8 +229,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", @@ -195,13 +246,13 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" dependencies = [ - "axum", - "axum-core", + "axum 0.7.5", + "axum-core 0.4.3", "bytes", "futures-util", "headers", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", @@ -220,10 +271,10 @@ dependencies = [ "arc-swap", "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-util", "pin-project-lite", "rustls 0.21.11", @@ -242,7 +293,7 @@ checksum = "7ea4cd08ae2a5f075d28fa31190163c8106a1d2d3189442494bae22b39040a0d" dependencies = [ "axum-server", "bytes", - "http", + "http 1.1.0", "http-body-util", "pin-project", "tokio", @@ -505,7 +556,7 @@ version = "0.3.0" dependencies = [ "argon2", "async-trait", - "axum", + "axum 0.7.5", "axum-extra", "axum-server", "axum-server-dual-protocol", @@ -513,6 +564,7 @@ dependencies = [ "bytes", "chrono", "clap", + "console-subscriber", "cyborgtime", "either", "figment", @@ -521,9 +573,9 @@ dependencies = [ "hickory-resolver", "hmac", "hot-lib-reloader", - "http", + "http 1.1.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-util", "image", "ipaddress", @@ -573,6 +625,42 @@ dependencies = [ "webpage", ] +[[package]] +name = "console-api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -917,6 +1005,20 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -1035,6 +1137,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.4" @@ -1046,8 +1167,8 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", - "indexmap", + "http 1.1.0", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1060,6 +1181,12 @@ version = "0.1.2+12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "647deb1583b14d160f85f3ff626f20b6edd366e3852c9843b06077388f794cb6" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.3" @@ -1076,7 +1203,20 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" dependencies = [ - "hashbrown", + "hashbrown 0.14.3", +] + +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", ] [[package]] @@ -1088,7 +1228,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 1.1.0", "httpdate", "mime", "sha1", @@ -1100,7 +1240,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http", + "http 1.1.0", ] [[package]] @@ -1242,6 +1382,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.1.0" @@ -1253,6 +1404,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.0" @@ -1260,7 +1422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -1271,8 +1433,8 @@ checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -1288,6 +1450,36 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.3.1" @@ -1297,9 +1489,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", "httparse", "httpdate", "itoa", @@ -1316,8 +1508,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.1.0", + "hyper 1.3.1", "hyper-util", "rustls 0.22.4", "rustls-pki-types", @@ -1326,6 +1518,18 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.28", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-util" version = "0.1.3" @@ -1335,9 +1539,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", "pin-project-lite", "socket2", "tokio", @@ -1393,6 +1597,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -1400,7 +1614,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.3", "serde", ] @@ -1477,6 +1691,15 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.11.0" @@ -2016,7 +2239,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap", + "indexmap 2.2.6", "js-sys", "once_cell", "pin-project-lite", @@ -2340,6 +2563,38 @@ dependencies = [ "yansi", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2451,10 +2706,10 @@ dependencies = [ "futures-core", "futures-util", "hickory-resolver", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-rustls", "hyper-util", "ipnet", @@ -2551,7 +2806,7 @@ dependencies = [ "assign", "bytes", "date_header", - "http", + "http 1.1.0", "js_int", "js_option", "maplit", @@ -2573,8 +2828,8 @@ dependencies = [ "base64 0.21.7", "bytes", "form_urlencoded", - "http", - "indexmap", + "http 1.1.0", + "indexmap 2.2.6", "js_int", "konst", "percent-encoding", @@ -2600,7 +2855,7 @@ version = "0.27.11" source = "git+https://github.com/girlbossceo/ruma?branch=conduwuit-changes#7136799881d0fcb18aad26c3dac18a34a1f892fc" dependencies = [ "as_variant", - "indexmap", + "indexmap 2.2.6", "js_int", "js_option", "percent-encoding", @@ -3020,7 +3275,7 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df141464944fdf8e2a6f2184eb1d973a20456466f788346b6e3a51791cdaa370" dependencies = [ - "http", + "http 1.1.0", "pin-project", "sentry-core", "tower-layer", @@ -3084,7 +3339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -3148,7 +3403,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -3516,9 +3771,20 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.2.0" @@ -3615,13 +3881,41 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" dependencies = [ - "indexmap", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -3630,9 +3924,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -3649,8 +3947,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", "pin-project-lite", "tokio", @@ -3676,8 +3974,7 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "log", "pin-project-lite", @@ -3688,8 +3985,7 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "proc-macro2", "quote", @@ -3699,8 +3995,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "once_cell", "valuable", @@ -3728,6 +4023,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-opentelemetry" version = "0.22.0" @@ -3741,7 +4046,7 @@ dependencies = [ "smallvec", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "tracing-subscriber", "web-time 0.2.4", ] @@ -3749,8 +4054,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "matchers", "nu-ansi-term", @@ -3761,7 +4065,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0 (git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index eb5384029..b7a14d3a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,8 @@ rust-version = "1.75.0" [dependencies] +console-subscriber = { version = "0.1", optional = true } + hot-lib-reloader = { version = "^0.6", optional = true } # Used for secure identifiers @@ -337,6 +339,18 @@ hardened_malloc-rs = { version = "0.1.2", optional = true, features = [ #hardened_malloc-rs = { optional = true, features = ["static","clang","light"], path = "../hardened_malloc-rs", default-features = false } +# backport of [https://github.com/tokio-rs/tracing/pull/2956] to the 0.1.x branch of tracing. +# we can switch back to upstream if #2956 is merged and backported in the upstream repo. +[patch.crates-io.tracing-subscriber] +git = "https://github.com/Benjamin-L/tracing" +branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +[patch.crates-io.tracing] +git = "https://github.com/Benjamin-L/tracing" +branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +[patch.crates-io.tracing-core] +git = "https://github.com/Benjamin-L/tracing" +branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" + [features] default = [ "backend_rocksdb", @@ -372,6 +386,10 @@ perf_measurements = [ "opentelemetry-jaeger", ] +# enable the tokio_console server +# incompatible with release_max_log_level +tokio_console = ["console-subscriber", "tokio/tracing"] + hot_reload = ["dep:hot-lib-reloader"] hardened_malloc = ["hardened_malloc-rs"] diff --git a/docs/development.md b/docs/development.md index d6ee27eb0..49e055e10 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,3 +2,21 @@ Information about developing the project. If you are only interested in using it, you can safely ignore this section. + +## Debugging with `tokio-console` + +[`tokio-console`][1] can be a useful tool for debugging and profiling. To make +a `tokio-console`-enabled build of Conduwuit, enable the `tokio_console` feature, +disable the default `release_max_log_level` feature, and set the +`--cfg tokio_unstable` flag to enable experimental tokio APIs. A build might +look like this: + +```bash +RUSTFLAGS="--cfg tokio_unstable" cargo build \ + --release \ + --no-default-features \ + --features + backend_rocksdb,systemd,element_hacks,sentry_telemetry,gzip_compression,brotli_compression,zstd_compression,tokio_console +``` + +[1]: https://docs.rs/tokio-console/latest/tokio_console/ diff --git a/src/database/mod.rs b/src/database/mod.rs index c8e1a17bd..e31e0a78a 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -40,8 +40,8 @@ use tokio::time::{interval, Instant}; use tracing::{debug, error, warn}; use crate::{ - database::migrations::migrations, service::rooms::timeline::PduCount, services, Config, Error, Result, Services, - SERVICES, + database::migrations::migrations, service::rooms::timeline::PduCount, services, Config, Error, + LogLevelReloadHandles, Result, Services, SERVICES, }; pub(crate) struct KeyValueDatabase { @@ -203,13 +203,7 @@ struct CheckForUpdatesResponse { impl KeyValueDatabase { /// Load an existing database or create a new one. #[allow(clippy::too_many_lines)] - pub(crate) async fn load_or_create( - config: Config, - tracing_reload_handler: tracing_subscriber::reload::Handle< - tracing_subscriber::EnvFilter, - tracing_subscriber::Registry, - >, - ) -> Result<()> { + pub(crate) async fn load_or_create(config: Config, tracing_reload_handler: LogLevelReloadHandles) -> Result<()> { Self::check_db_setup(&config)?; if !Path::new(&config.database_path).exists() { diff --git a/src/main.rs b/src/main.rs index 6773ce9ab..a1676cad5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,7 @@ use std::os::unix::fs::PermissionsExt as _; /* not unix specific, just only for // Not async due to services() being used in many closures, and async closures // are not stable as of writing This is the case for every other occurence of // sync Mutex/RwLock, except for database related ones -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use std::{any::Any, io, net::SocketAddr, sync::atomic, time::Duration}; use api::ruma_wrapper::{Ruma, RumaResponse}; @@ -56,6 +56,12 @@ mod routes; mod service; mod utils; +#[cfg(all(feature = "tokio_console", feature = "release_max_log_level"))] +compile_error!( + "'tokio_console' feature and 'release_max_log_level' feature are incompatible, because console-subscriber needs \ + access to trace-level events. 'release_max_log_level' must be disabled to use tokio-console" +); + pub(crate) static SERVICES: RwLock>> = RwLock::new(None); #[must_use] @@ -79,7 +85,7 @@ struct Server { runtime: tokio::runtime::Runtime, - tracing_reload_handle: reload::Handle, + tracing_reload_handle: LogLevelReloadHandles, #[cfg(feature = "sentry_telemetry")] _sentry_guard: Option, @@ -547,7 +553,56 @@ fn init_sentry(config: &Config) -> sentry::ClientInitGuard { )) } -fn init_tracing_sub(config: &Config) -> reload::Handle { +// We need to store a reload::Handle value, but can't name it's type explicitly +// because the S type parameter depends on the subscriber's previous layers. In +// our case, this includes unnameable 'impl Trait' types. +// +// This is fixed[1] in the unreleased tracing-subscriber from the master branch, +// which removes the S parameter. Unfortunately can't use it without pulling in +// a version of tracing that's incompatible with the rest of our deps. +// +// To work around this, we define an trait without the S paramter that forwards +// to the reload::Handle::reload method, and then store the handle as a trait +// object. +// +// [1]: https://github.com/tokio-rs/tracing/pull/1035/commits/8a87ea52425098d3ef8f56d92358c2f6c144a28f +trait ReloadHandle { + fn reload(&self, new_value: L) -> Result<(), reload::Error>; +} + +impl ReloadHandle for reload::Handle { + fn reload(&self, new_value: L) -> Result<(), reload::Error> { reload::Handle::reload(self, new_value) } +} + +struct LogLevelReloadHandlesInner { + handles: Vec + Send + Sync>>, +} + +/// Wrapper to allow reloading the filter on several several +/// [`tracing_subscriber::reload::Handle`]s at once, with the same value. +#[derive(Clone)] +struct LogLevelReloadHandles { + inner: Arc, +} + +impl LogLevelReloadHandles { + fn new(handles: Vec + Send + Sync>>) -> LogLevelReloadHandles { + LogLevelReloadHandles { + inner: Arc::new(LogLevelReloadHandlesInner { + handles, + }), + } + } + + fn reload(&self, new_value: &EnvFilter) -> Result<(), reload::Error> { + for handle in &self.inner.handles { + handle.reload(new_value.clone())?; + } + Ok(()) + } +} + +fn init_tracing_sub(config: &Config) -> LogLevelReloadHandles { let registry = Registry::default(); let fmt_layer = tracing_subscriber::fmt::Layer::new(); let filter_layer = match EnvFilter::try_new(&config.log) { @@ -558,35 +613,34 @@ fn init_tracing_sub(config: &Config) -> reload::Handle { }, }; - let (reload_filter, reload_handle) = reload::Layer::new(filter_layer); + let mut reload_handles = Vec:: + Send + Sync>>::new(); + let subscriber = registry; - #[cfg(feature = "sentry_telemetry")] - let sentry_layer = sentry_tracing::layer(); + #[cfg(feature = "tokio_console")] + let subscriber = { + let console_layer = console_subscriber::spawn(); + subscriber.with(console_layer) + }; - let subscriber; + let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); + reload_handles.push(Box::new(fmt_reload_handle)); + let subscriber = subscriber.with(fmt_layer.with_filter(fmt_reload_filter)); - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental #[cfg(feature = "sentry_telemetry")] - { - subscriber = registry - .with(reload_filter) - .with(fmt_layer) - .with(sentry_layer); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sentry_telemetry"))] - { - subscriber = registry.with(reload_filter).with(fmt_layer); + let subscriber = { + let sentry_layer = sentry_tracing::layer(); + let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer); + reload_handles.push(Box::new(sentry_reload_handle)); + subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; tracing::subscriber::set_global_default(subscriber).unwrap(); - reload_handle + LogLevelReloadHandles::new(reload_handles) } #[cfg(feature = "perf_measurements")] -fn init_tracing_jaeger(config: &Config) -> reload::Handle { +fn init_tracing_jaeger(config: &Config) -> LogLevelReloadHandles { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_auto_split_batch(true) @@ -609,11 +663,12 @@ fn init_tracing_jaeger(config: &Config) -> reload::Handle { tracing::subscriber::set_global_default(subscriber).unwrap(); - reload_handle + LogLevelReloadHandles::new(vec![Box::new(reload_handle)]) } +// TODO: tokio-console here? #[cfg(feature = "perf_measurements")] -fn init_tracing_flame(_config: &Config) -> reload::Handle { +fn init_tracing_flame(_config: &Config) -> LogLevelReloadHandles { let registry = Registry::default(); let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap(); let flame_layer = flame_layer.with_empty_samples(false); @@ -626,7 +681,7 @@ fn init_tracing_flame(_config: &Config) -> reload::Handle { tracing::subscriber::set_global_default(subscriber).unwrap(); - reload_handle + LogLevelReloadHandles::new(vec![Box::new(reload_handle)]) } // This is needed for opening lots of file descriptors, which tends to diff --git a/src/service/admin/debug/debug_commands.rs b/src/service/admin/debug/debug_commands.rs index b3c84e40c..c91c8f4fe 100644 --- a/src/service/admin/debug/debug_commands.rs +++ b/src/service/admin/debug/debug_commands.rs @@ -331,7 +331,7 @@ pub(crate) async fn change_log_level( match services() .globals .tracing_reload_handle - .modify(|filter| *filter = old_filter_layer) + .reload(&old_filter_layer) { Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( @@ -360,7 +360,7 @@ pub(crate) async fn change_log_level( match services() .globals .tracing_reload_handle - .modify(|filter| *filter = new_filter_layer) + .reload(&new_filter_layer) { Ok(()) => { return Ok(RoomMessageEventContent::text_plain("Successfully changed log level")); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index a28d22214..f3688838e 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -27,10 +27,9 @@ use ruma::{ }; use tokio::sync::{broadcast, watch::Receiver, Mutex, RwLock, Semaphore}; use tracing::{error, info, trace}; -use tracing_subscriber::{EnvFilter, Registry}; use url::Url; -use crate::{services, Config, Result}; +use crate::{services, Config, LogLevelReloadHandles, Result}; mod client; mod data; @@ -45,7 +44,7 @@ type SyncHandle = ( pub(crate) struct Service<'a> { pub(crate) db: &'static dyn Data, - pub(crate) tracing_reload_handle: tracing_subscriber::reload::Handle, + pub(crate) tracing_reload_handle: LogLevelReloadHandles, pub(crate) config: Config, pub(crate) cidr_range_denylist: Vec, keypair: Arc, @@ -100,8 +99,7 @@ impl Default for RotationHandler { impl Service<'_> { pub(crate) fn load( - db: &'static dyn Data, config: &Config, - tracing_reload_handle: tracing_subscriber::reload::Handle, + db: &'static dyn Data, config: &Config, tracing_reload_handle: LogLevelReloadHandles, ) -> Result { let keypair = db.load_keypair(); diff --git a/src/service/mod.rs b/src/service/mod.rs index e530a2bb4..846d5ff54 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -6,7 +6,7 @@ use std::{ use lru_cache::LruCache; use tokio::sync::{broadcast, Mutex, RwLock}; -use crate::{Config, Result}; +use crate::{Config, LogLevelReloadHandles, Result}; pub(crate) mod account_data; pub(crate) mod admin; @@ -55,11 +55,7 @@ impl Services<'_> { + sending::Data + 'static, >( - db: &'static D, config: &Config, - tracing_reload_handle: tracing_subscriber::reload::Handle< - tracing_subscriber::EnvFilter, - tracing_subscriber::Registry, - >, + db: &'static D, config: &Config, tracing_reload_handle: LogLevelReloadHandles, ) -> Result { Ok(Self { appservice: appservice::Service::build(db)?,