From c7d9a9156c1a4e9e588f1bbee4a37f1b8f33983f Mon Sep 17 00:00:00 2001 From: Alejandro Pedraza Date: Tue, 12 Dec 2023 14:56:20 -0500 Subject: [PATCH] Introduce channel, metrics, dep updates --- Cargo.lock | 468 +++++++++------------------------- deny.toml | 3 - reinitialize-pods/Cargo.toml | 5 +- reinitialize-pods/src/lib.rs | 178 +++++++++---- reinitialize-pods/src/main.rs | 15 +- validator/Cargo.toml | 2 +- 6 files changed, 257 insertions(+), 414 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74db0ba4..c23a715d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,7 +86,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -127,26 +127,6 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" -[[package]] -name = "bindgen" -version = "0.68.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" -dependencies = [ - "bitflags 2.4.1", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.39", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -180,15 +160,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -208,17 +179,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "clang-sys" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.4.11" @@ -248,7 +208,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -282,28 +242,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-epoch" -version = "0.9.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] - [[package]] name = "deflate" version = "1.0.0" @@ -334,6 +272,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "dtoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" + [[package]] name = "either" version = "1.9.0" @@ -442,7 +386,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -492,12 +436,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "gzip-header" version = "1.0.0" @@ -507,21 +445,6 @@ dependencies = [ "crc32fast", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" -dependencies = [ - "ahash", -] - [[package]] name = "hashbrown" version = "0.14.3" @@ -572,9 +495,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -691,16 +614,6 @@ dependencies = [ "cc", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.1.0" @@ -708,7 +621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown", ] [[package]] @@ -722,9 +635,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" @@ -850,7 +763,7 @@ dependencies = [ "backoff", "derivative", "futures", - "hashbrown 0.14.3", + "hashbrown", "json-patch", "k8s-openapi", "kube-client", @@ -867,9 +780,9 @@ dependencies = [ [[package]] name = "kubert" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5f21c2e19593cc66c008c70a948da44c2bdedd8d56747e4eeb8b21f8a656fd" +checksum = "a4c7e2b9edcf6c5848ce9819712d927a5c8fbd63d548f817ca64022444e7cdd1" dependencies = [ "ahash", "bytes", @@ -882,9 +795,10 @@ dependencies = [ "kube-client", "kube-core", "kube-runtime", - "metrics-exporter-prometheus", - "metrics-process", + "kubert-prometheus-process", + "kubert-prometheus-tokio", "pin-project-lite", + "prometheus-client", "serde", "thiserror", "tokio", @@ -895,43 +809,40 @@ dependencies = [ ] [[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" +name = "kubert-prometheus-process" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +checksum = "40d2f45f0fb66a3681630fe2efd018c63eb7336697c44d909e53a99847113b97" +dependencies = [ + "libc", + "procfs", + "prometheus-client", + "tracing", +] [[package]] -name = "libc" -version = "0.2.150" +name = "kubert-prometheus-tokio" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "a101fa3df488c89130664aaf4652986da49e204fb1725d089122f75b22ff6cbb" +dependencies = [ + "prometheus-client", + "tokio", + "tokio-metrics", + "tracing", +] [[package]] -name = "libloading" -version = "0.7.4" +name = "lazy_static" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "libproc" -version = "0.14.2" +name = "libc" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229004ebba9d1d5caf41623f1523b6d52abb47d9f6ab87f7e6fc992e3b854aef" -dependencies = [ - "bindgen", - "errno", - "libc", -] +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "linked-hash-map" @@ -972,6 +883,7 @@ dependencies = [ "kube", "kubert", "openssl", + "prometheus-client", "tokio", "tracing", ] @@ -998,15 +910,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "mach2" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" -dependencies = [ - "libc", -] - [[package]] name = "matchers" version = "0.1.0" @@ -1022,93 +925,12 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - -[[package]] -name = "metrics" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" -dependencies = [ - "ahash", - "metrics-macros", - "portable-atomic", -] - -[[package]] -name = "metrics-exporter-prometheus" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" -dependencies = [ - "base64", - "indexmap 1.9.3", - "metrics", - "metrics-util", - "quanta", - "thiserror", -] - -[[package]] -name = "metrics-macros" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "metrics-process" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674a02f6ad51326c2106d9aa5a07d1f759695b655c06df0bba5d5fb338ac0a4" -dependencies = [ - "libproc", - "mach2", - "metrics", - "once_cell", - "procfs", - "rlimit", - "windows", -] - -[[package]] -name = "metrics-util" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "hashbrown 0.13.1", - "metrics", - "num_cpus", - "quanta", - "sketches-ddsketch", -] - [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.7.1" @@ -1129,16 +951,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num-traits" version = "0.2.17" @@ -1169,9 +981,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" @@ -1196,7 +1008,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1207,9 +1019,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.6+3.1.4" +version = "300.2.1+3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" +checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" dependencies = [ "cc", ] @@ -1259,17 +1071,11 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pem" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" +checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ "base64", "serde", @@ -1298,7 +1104,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1319,12 +1125,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1364,19 +1164,26 @@ dependencies = [ ] [[package]] -name = "quanta" -version = "0.11.1" +name = "prometheus-client" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +checksum = "510c4f1c9d81d556458f94c98f857748130ea9737bbd6053da497503b26ea63c" dependencies = [ - "crossbeam-utils", - "libc", - "mach2", - "once_cell", - "raw-cpuid", - "wasi", - "web-sys", - "winapi", + "dtoa", + "itoa", + "parking_lot", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.40", ] [[package]] @@ -1418,15 +1225,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -1494,32 +1292,17 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "rlimit" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3560f70f30a0f16d11d01ed078a07740fe6b489667abc7c7b029155d9f21c3d8" -dependencies = [ - "libc", -] - [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustix" -version = "0.38.26" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", @@ -1530,9 +1313,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.9" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring", @@ -1573,9 +1356,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" @@ -1662,7 +1445,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1671,7 +1454,7 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ - "indexmap 2.1.0", + "indexmap", "itoa", "ryu", "serde", @@ -1683,7 +1466,7 @@ version = "0.9.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" dependencies = [ - "indexmap 2.1.0", + "indexmap", "itoa", "ryu", "serde", @@ -1699,12 +1482,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shlex" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" - [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1714,12 +1491,6 @@ dependencies = [ "libc", ] -[[package]] -name = "sketches-ddsketch" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" - [[package]] name = "slab" version = "0.4.9" @@ -1774,9 +1545,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "13fa70a4ee923979ffb522cacce59d34421ebdea5625e1073c4326ef9d2dd42e" dependencies = [ "proc-macro2", "quote", @@ -1800,7 +1571,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1815,9 +1586,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" dependencies = [ "backtrace", "bytes", @@ -1850,14 +1621,26 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", +] + +[[package]] +name = "tokio-metrics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", ] [[package]] name = "tokio-openssl" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +checksum = "6ffab79df67727f6acf57f1ff743091873c24c579b1e2ce4d8f53e47ded4d63d" dependencies = [ "futures-util", "openssl", @@ -1875,6 +1658,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.10" @@ -1960,7 +1754,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -2025,9 +1819,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicode-ident" @@ -2101,7 +1895,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", "wasm-bindgen-shared", ] @@ -2123,7 +1917,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2134,16 +1928,6 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" -[[package]] -name = "web-sys" -version = "0.3.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "winapi" version = "0.3.9" @@ -2166,16 +1950,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.51.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" -dependencies = [ - "windows-core", - "windows-targets 0.48.5", -] - [[package]] name = "windows-core" version = "0.51.1" @@ -2319,22 +2093,22 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "zerocopy" -version = "0.7.29" +version = "0.7.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e" +checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.29" +version = "0.7.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2" +checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] diff --git a/deny.toml b/deny.toml index db60c24d..0301092c 100644 --- a/deny.toml +++ b/deny.toml @@ -59,9 +59,6 @@ skip = [ # `rustls-pemfile` and `k8s-openapi` depend on versions of `base64` that # have diverged significantly. { name = "base64" }, - # `tower-http` (a transitive dep via `kubert`) depends on v2.x of `bitflags`, - # while pretty much the entire rest of the world is still on v1.x - { name = "bitflags", version = "1.0" }, # https://github.com/hawkw/matchers/pull/4 { name = "regex-automata", version = "0.1" }, { name = "regex-syntax", version = "0.6" }, diff --git a/reinitialize-pods/Cargo.toml b/reinitialize-pods/Cargo.toml index d0b6f024..65ddc237 100644 --- a/reinitialize-pods/Cargo.toml +++ b/reinitialize-pods/Cargo.toml @@ -19,6 +19,7 @@ futures-util = "0.3" k8s-openapi = { version = "0.20", features = ["v1_22"] } kube = { version = "0.87", features = ["runtime"] } openssl = { version = "0.10.60", optional = true } +prometheus-client = "0.22" tracing = "0.1" [dependencies.clap] @@ -27,9 +28,9 @@ default-features = false features = ["derive", "env", "help", "std"] [dependencies.kubert] -version = "0.20.0" +version = "0.21" default-features = false -features = ["admin", "clap", "metrics", "runtime"] +features = ["admin", "clap", "prometheus-client", "runtime"] [dependencies.tokio] version = "1" diff --git a/reinitialize-pods/src/lib.rs b/reinitialize-pods/src/lib.rs index 2512fc1f..75f5bc25 100644 --- a/reinitialize-pods/src/lib.rs +++ b/reinitialize-pods/src/lib.rs @@ -1,89 +1,137 @@ -use futures_util::StreamExt; +use futures_util::{Stream, StreamExt}; use k8s_openapi::api::core::v1::Pod; use kube::{ runtime::{ events::{Event, EventType, Recorder, Reporter}, watcher, }, - Client, Error, Resource, + Client, Error, Resource, ResourceExt, }; use kubert::Runtime; +use prometheus_client::{metrics::counter::Counter, registry::Registry}; +use tokio::sync::mpsc::{self, error::TrySendError, Receiver, Sender}; // ERRNO 95: Operation not supported pub const UNSUCCESSFUL_EXIT_CODE: i32 = 95; +// If the event channel capacity is reached, the event is dropped, but a new one will be emitted +// in the pod's next crashloop iteration +const EVENT_CHANNEL_CAPACITY: usize = 32; + const DATA_PLANE_LABEL: &str = "linkerd.io/control-plane-ns"; const CONDITION_EVICTED_REASON: &str = "EvictionByEvictionAPI"; const EVENT_ACTION: &str = "Evicting"; const EVENT_REASON: &str = "LinkerdCNINotConfigured"; -pub fn run(runtime: &mut Runtime, node_name: String, controller_pod_name: String) { - let client = runtime.client().clone(); - let pod_evts = runtime.watch_all::( +#[derive(Clone, Debug)] +pub struct Metrics { + queue_overflow: Counter, + evicted_pods: Counter, +} + +pub fn run(rt: &mut Runtime, node_name: String, controller_pod_name: String, metrics: Metrics) { + let pod_evts = rt.watch_all::( watcher::Config::default() .labels(DATA_PLANE_LABEL) .fields(&format!("spec.nodeName={node_name}")), ); + let (tx, rx) = mpsc::channel(EVENT_CHANNEL_CAPACITY); + tokio::spawn(process_events(pod_evts, tx, metrics.clone())); + + let client = rt.client(); + tokio::spawn(process_pods(client, controller_pod_name, rx, metrics)); +} - tokio::spawn(async move { - tokio::pin!(pod_evts); - while let Some(evt) = pod_evts.next().await { - if let watcher::Event::Applied(pod) = evt { - let status = if let Some(ref status) = pod.status { - status.clone() - } else { - tracing::info!("Skipped, no status"); - continue; - }; +async fn process_events( + pod_evts: impl Stream>, + tx: Sender, + metrics: Metrics, +) { + tokio::pin!(pod_evts); + while let Some(evt) = pod_evts.next().await { + tracing::trace!(?evt); + if let watcher::Event::Applied(pod) = evt { + let status = if let Some(ref status) = pod.status { + status.clone() + } else { + tracing::info!("Skipped, no status"); + continue; + }; - let terminated = status - .init_container_statuses - .unwrap_or_default() - .iter() - .find(|status| status.name == "linkerd-network-validator") - .and_then(|status| status.last_state.as_ref()) - .and_then(|status| status.terminated.as_ref()) - .map(|term| term.exit_code == UNSUCCESSFUL_EXIT_CODE) - .unwrap_or(false); + let terminated = status + .init_container_statuses + .unwrap_or_default() + .iter() + .find(|status| status.name == "linkerd-network-validator") + .and_then(|status| status.last_state.as_ref()) + .and_then(|status| status.terminated.as_ref()) + .map(|term| term.exit_code == UNSUCCESSFUL_EXIT_CODE) + .unwrap_or(false); - let evicted = status - .conditions - .as_ref() - .and_then(|conds| { - conds.iter().find(|cond| { - cond.reason - .as_ref() - .is_some_and(|reason| reason == CONDITION_EVICTED_REASON) - }) + let evicted = status + .conditions + .as_ref() + .and_then(|conds| { + conds.iter().find(|cond| { + cond.reason + .as_ref() + .is_some_and(|reason| reason == CONDITION_EVICTED_REASON) }) - .is_some(); + }) + .is_some(); - if terminated && !evicted { - let namespace = pod.metadata.namespace.as_ref().unwrap(); - let name = pod.metadata.name.as_ref().unwrap(); - let pods = kube::Api::::namespaced(client.clone(), namespace); - let evict_res = pods.evict(name, &Default::default()).await; - match evict_res { - Ok(_) => { - tracing::info!(name = format!("{namespace}/{name}"), "Evicting pod"); - if let Err(err) = - publish_event(client.clone(), controller_pod_name.clone(), &pod) - .await - { - tracing::warn!(%err, name = format!("{namespace}/{name}"), "Error publishing event"); - } - } - Err(err) => { - tracing::warn!(%err, name = format!("{namespace}/{name}"), "Error evicting pod") - } + if terminated && !evicted { + let namespace = pod.namespace().unwrap(); + let name = pod.name_any(); + // this avoids blocking the event loop + match tx.try_send(pod.clone()) { + Ok(_) => {} + Err(TrySendError::Full(_)) => { + tracing::warn!( + name = format!("{namespace}/{name}"), + "Dropped event (channel full)" + ); + metrics.queue_overflow.inc(); } + Err(TrySendError::Closed(_)) => tracing::warn!( + name = format!("{namespace}/{name}"), + "Dropped event (channel closed or dropped)" + ), } } } - }); + } } -async fn publish_event( +async fn process_pods( + client: Client, + controller_pod_name: String, + mut rx: Receiver, + metrics: Metrics, +) { + while let Some(pod) = rx.recv().await { + let namespace = pod.namespace().unwrap(); + let name = pod.name_any(); + let pods = kube::Api::::namespaced(client.clone(), &namespace); + let evict_res = pods.evict(&name, &Default::default()).await; + match evict_res { + Ok(_) => { + tracing::info!(name = format!("{namespace}/{name}"), "Evicting pod"); + metrics.evicted_pods.inc(); + if let Err(err) = + publish_k8s_event(client.clone(), controller_pod_name.clone(), &pod).await + { + tracing::warn!(%err, name = format!("{namespace}/{name}"), "Error publishing event"); + } + } + Err(err) => { + tracing::warn!(%err, name = format!("{namespace}/{name}"), "Error evicting pod") + } + } + } +} + +async fn publish_k8s_event( client: Client, controller_pod_name: String, pod: &Pod, @@ -104,3 +152,25 @@ async fn publish_event( }) .await } + +impl Metrics { + pub fn register(prom: &mut Registry) -> Self { + let queue_overflow = Counter::::default(); + prom.register( + "queue_overflow", + "Incremented whenever the event processing queue overflows", + queue_overflow.clone(), + ); + let evicted_pods = Counter::::default(); + prom.register( + "evicted", + "Number of pods evicted by the controller", + evicted_pods.clone(), + ); + + Self { + queue_overflow, + evicted_pods, + } + } +} diff --git a/reinitialize-pods/src/main.rs b/reinitialize-pods/src/main.rs index 21303eae..7947454b 100644 --- a/reinitialize-pods/src/main.rs +++ b/reinitialize-pods/src/main.rs @@ -1,5 +1,7 @@ use anyhow::{bail, Result}; use clap::Parser; +use kubert::Runtime; +use linkerd_reinitialize_pods::Metrics; /// Scan the pods in the current node to find those that have been injected by linkerd, and whose /// linkerd-network-validator container has failed, and proceed to evict them so they can restart @@ -46,21 +48,20 @@ async fn main() -> Result<()> { admin, } = Args::parse(); - let mut admin = admin.into_builder(); - admin.with_default_prometheus(); - - let mut runtime = kubert::Runtime::builder() + let mut prom = prometheus_client::registry::Registry::default(); + let metrics = Metrics::register(prom.sub_registry_with_prefix("linkerd_reinitialize_pods")); + let mut rt = Runtime::builder() .with_log(log_level, log_format) - .with_admin(admin) + .with_admin(admin.into_builder().with_prometheus(prom)) .with_client(client) .build() .await?; - linkerd_reinitialize_pods::run(&mut runtime, node_name, controller_pod_name); + linkerd_reinitialize_pods::run(&mut rt, node_name, controller_pod_name, metrics); // Block the main thread on the shutdown signal. Once it fires, wait for the background tasks to // complete before exiting. - if runtime.run().await.is_err() { + if rt.run().await.is_err() { bail!("aborted"); } diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 751b9c3b..15ed9243 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -9,7 +9,7 @@ publish = false [dependencies] anyhow = "1" bytes = "1" -kubert = { version = "0.20.0", features = ["log"] } +kubert = { version = "0.21", features = ["log"] } rand = "0.8" tracing = "0.1"