From efa275b17da862c55d22ac2b837fc2cc364f27f0 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 7 Apr 2025 21:12:51 +0200 Subject: [PATCH 1/7] start watching configmaps that are referenced in hive spec --- rust/operator-binary/src/main.rs | 49 ++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 583941cb..efd5295e 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -12,22 +12,15 @@ use std::sync::Arc; use clap::{Parser, crate_description, crate_version}; use futures::stream::StreamExt; use stackable_operator::{ - YamlSchema, - cli::{Command, ProductOperatorRun}, - k8s_openapi::api::{ + cli::{Command, ProductOperatorRun}, k8s_openapi::api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, Service}, - }, - kube::{ + }, kube::{ core::DeserializeGuard, runtime::{ - Controller, - events::{Recorder, Reporter}, - watcher, - }, - }, - logging::controller::report_controller_reconciled, - shared::yaml::SerializeOptions, + events::{Recorder, Reporter}, reflector::ObjectRef, watcher, Controller + }, ResourceExt, + }, logging::controller::report_controller_reconciled, shared::yaml::SerializeOptions, YamlSchema }; use crate::{ @@ -89,11 +82,12 @@ async fn main() -> anyhow::Result<()> { instance: None, })); - Controller::new( + let hive_controller = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default(), - ) - .owns( + ); + let hive_store_1 = hive_controller.store(); + hive_controller.owns( watch_namespace.get_api::(&client), watcher::Config::default(), ) @@ -106,6 +100,17 @@ async fn main() -> anyhow::Result<()> { watcher::Config::default(), ) .shutdown_on_signal() + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + move |config_map| { + hive_store_1 + .state() + .into_iter() + .filter(move |hive| references_config_map(hive, &config_map)) + .map(|hive| ObjectRef::from_obj(&*hive)) + }, + ) .run( controller::reconcile_hive, controller::error_policy, @@ -137,3 +142,17 @@ async fn main() -> anyhow::Result<()> { Ok(()) } + +fn references_config_map( + hive: &DeserializeGuard, + config_map: &DeserializeGuard, +) -> bool { + let Ok(hive) = &hive.0 else { + return false; + }; + + match hive.spec.cluster_config.hdfs.to_owned() { + Some(hdfs_connection) => hdfs_connection.config_map == config_map.name_any(), + None => false, + } +} \ No newline at end of file From 4cf9fd27175c3fa7dedeb3b40957959bcb26ac28 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 7 Apr 2025 21:13:58 +0200 Subject: [PATCH 2/7] wip: Inject the vector aggregator address into the vector config using an env var --- Cargo.lock | 401 +++++++++++++------- Cargo.toml | 3 +- rust/operator-binary/src/controller.rs | 28 +- rust/operator-binary/src/product_logging.rs | 44 +-- 4 files changed, 278 insertions(+), 198 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25ed8b9e..e336b103 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy 0.7.35", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.87" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", @@ -173,11 +173,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ - "getrandom", + "getrandom 0.2.15", "instant", "rand", ] +[[package]] +name = "backon" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970d91570c01a8a5959b36ad7dd1c30642df24b6b3068710066f6809f7033bb7" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -271,9 +282,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.16" +version = "1.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c" dependencies = [ "jobserver", "libc", @@ -301,9 +312,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.31" +version = "4.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" +checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" dependencies = [ "clap_builder", "clap_derive", @@ -311,9 +322,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.31" +version = "4.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" +checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" dependencies = [ "anstream", "anstyle", @@ -323,9 +334,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.28" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "heck", "proc-macro2", @@ -445,9 +456,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -455,9 +466,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", @@ -469,9 +480,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", @@ -480,9 +491,9 @@ dependencies = [ [[package]] name = "delegate" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297806318ef30ad066b15792a8372858020ae3ca2e414ee6c2133b1eb9e9e945" +checksum = "b9b6483c2bbed26f97861cf57651d4f2b731964a28cd2257f934a4b452480d21" dependencies = [ "proc-macro2", "quote", @@ -491,9 +502,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] @@ -620,9 +631,9 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ "event-listener", "pin-project-lite", @@ -639,6 +650,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "fluent-uri" version = "0.1.4" @@ -656,9 +673,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "form_urlencoded" @@ -789,7 +806,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -800,9 +829,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "git2" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff" +checksum = "5220b8ba44c68a9a7f7a7659e864dd73692e417ef0211bea133c7b74e031eeb9" dependencies = [ "bitflags 2.9.0", "libc", @@ -817,6 +846,18 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -896,9 +937,9 @@ dependencies = [ [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -917,12 +958,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", + "futures-core", "http", "http-body", "pin-project-lite", @@ -961,9 +1002,9 @@ dependencies = [ [[package]] name = "hyper-http-proxy" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d06dbdfbacf34d996c6fb540a71a684a7aae9056c71951163af8a8a4c07b9a4" +checksum = "7ad4b0a1e37510028bc4ba81d0e38d239c39671b0f0ce9e02dfa93a8133f7c08" dependencies = [ "bytes", "futures-util", @@ -1013,9 +1054,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", @@ -1023,6 +1064,7 @@ dependencies = [ "http", "http-body", "hyper", + "libc", "pin-project-lite", "socket2", "tokio", @@ -1032,16 +1074,17 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "windows-core", + "windows-core 0.61.0", ] [[package]] @@ -1094,9 +1137,9 @@ dependencies = [ [[package]] name = "icu_locid_transform_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" [[package]] name = "icu_normalizer" @@ -1118,9 +1161,9 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" [[package]] name = "icu_properties" @@ -1139,9 +1182,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" [[package]] name = "icu_provider" @@ -1200,9 +1243,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1263,10 +1306,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -1294,11 +1338,11 @@ dependencies = [ [[package]] name = "json-patch" -version = "3.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863726d7afb6bc2590eeff7135d923545e5e964f004c2ccf8716c25e70a86f08" +checksum = "159294d661a039f7644cea7e4d844e6b25aaf71c1ffe9d73a96d768c24b0faf4" dependencies = [ - "jsonptr 0.6.3", + "jsonptr 0.7.1", "serde", "serde_json", "thiserror 1.0.69", @@ -1345,9 +1389,9 @@ dependencies = [ [[package]] name = "jsonptr" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dea2b27dd239b2556ed7a25ba842fe47fd602e7fc7433c2a8d6106d4d9edd70" +checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe" dependencies = [ "serde", "serde_json", @@ -1406,15 +1450,15 @@ dependencies = [ [[package]] name = "kube" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32053dc495efad4d188c7b33cc7c02ef4a6e43038115348348876efd39a53cba" +checksum = "9a4eb20010536b48abe97fec37d23d43069bcbe9686adcf9932202327bc5ca6e" dependencies = [ "k8s-openapi 0.24.0", - "kube-client 0.98.0", - "kube-core 0.98.0", - "kube-derive 0.98.0", - "kube-runtime 0.98.0", + "kube-client 0.99.0", + "kube-core 0.99.0", + "kube-derive 0.99.0", + "kube-runtime 0.99.0", ] [[package]] @@ -1457,9 +1501,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d34ad38cdfbd1fa87195d42569f57bb1dda6ba5f260ee32fef9570b7937a0c9" +checksum = "7fc2ed952042df20d15ac2fe9614d0ec14b6118eab89633985d4b36e688dccf1" dependencies = [ "base64 0.22.1", "bytes", @@ -1477,10 +1521,9 @@ dependencies = [ "hyper-util", "jsonpath-rust 0.7.5", "k8s-openapi 0.24.0", - "kube-core 0.98.0", + "kube-core 0.99.0", "pem", "rustls", - "rustls-pemfile", "secrecy", "serde", "serde_json", @@ -1513,14 +1556,14 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97aa830b288a178a90e784d1b0f1539f2d200d2188c7b4a3146d9dc983d596f3" +checksum = "ff0d0793db58e70ca6d689489183816cb3aa481673e7433dc618cf7e8007c675" dependencies = [ "chrono", "form_urlencoded", "http", - "json-patch 3.0.1", + "json-patch 4.0.0", "k8s-openapi 0.24.0", "schemars", "serde", @@ -1544,13 +1587,14 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37745d8a4076b77e0b1952e94e358726866c8e14ec94baaca677d47dcdb98658" +checksum = "c562f58dc9f7ca5feac8a6ee5850ca221edd6f04ce0dd2ee873202a88cd494c9" dependencies = [ "darling", "proc-macro2", "quote", + "serde", "serde_json", "syn 2.0.100", ] @@ -1585,23 +1629,22 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a41af186a0fe80c71a13a13994abdc3ebff80859ca6a4b8a6079948328c135b" +checksum = "88f34cfab9b4bd8633062e0e85edb81df23cb09f159f2e31c60b069ae826ffdc" dependencies = [ "ahash", "async-broadcast", "async-stream", "async-trait", - "backoff", + "backon", "educe", "futures 0.3.31", "hashbrown 0.15.2", "hostname", - "json-patch 3.0.1", - "jsonptr 0.6.3", + "json-patch 4.0.0", "k8s-openapi 0.24.0", - "kube-client 0.98.0", + "kube-client 0.99.0", "parking_lot", "pin-project", "serde", @@ -1620,15 +1663,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.170" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libgit2-sys" -version = "0.18.0+1.9.0" +version = "0.18.1+1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec" +checksum = "e1dcb20f84ffcdd825c7a311ae347cce604a6f084a767dec4a4929829645290e" dependencies = [ "cc", "libc", @@ -1638,9 +1681,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.21" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" dependencies = [ "cc", "libc", @@ -1666,9 +1709,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "matchers" @@ -1693,9 +1736,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "ff70ce3e48ae43fa075863cef62e8b43b71a4f2382229920e0df362592919430" dependencies = [ "adler2", ] @@ -1707,7 +1750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -1757,9 +1800,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl-probe" @@ -1896,9 +1939,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.15" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" dependencies = [ "memchr", "thiserror 2.0.12", @@ -1907,9 +1950,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.15" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" +checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" dependencies = [ "pest", "pest_generator", @@ -1917,9 +1960,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.15" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" +checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" dependencies = [ "pest", "pest_meta", @@ -1930,9 +1973,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.15" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" +checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" dependencies = [ "once_cell", "pest", @@ -1989,7 +2032,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.23", + "zerocopy 0.8.24", ] [[package]] @@ -2028,13 +2071,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "rand" version = "0.8.5" @@ -2062,14 +2111,14 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags 2.9.0", ] @@ -2126,13 +2175,13 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "ring" -version = "0.17.13" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "untrusted", "windows-sys 0.52.0", @@ -2185,9 +2234,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.23" +version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ "log", "once_cell", @@ -2240,9 +2289,9 @@ checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ "ring", "rustls-pki-types", @@ -2475,9 +2524,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "snafu" @@ -2523,9 +2572,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2565,8 +2614,7 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.87.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" +version = "0.87.5" dependencies = [ "chrono", "clap", @@ -2577,9 +2625,9 @@ dependencies = [ "either", "futures 0.3.31", "indexmap", - "json-patch 3.0.1", + "json-patch 4.0.0", "k8s-openapi 0.24.0", - "kube 0.98.0", + "kube 0.99.0", "opentelemetry-jaeger", "opentelemetry_sdk", "product-config", @@ -2604,7 +2652,6 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "darling", "proc-macro2", @@ -2615,9 +2662,8 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ - "kube 0.98.0", + "kube 0.99.0", "semver", "serde", "serde_yaml", @@ -2795,9 +2841,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.39" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -2810,15 +2856,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -2836,9 +2882,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.0" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9975ea0f48b5aa3972bf2d888c238182458437cc2a19374b81b25cdf1023fb3a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", @@ -2886,9 +2932,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", @@ -3178,6 +3224,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -3274,7 +3329,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core", + "windows-core 0.52.0", "windows-targets", ] @@ -3287,11 +3342,64 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-link" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] [[package]] name = "windows-sys" @@ -3377,13 +3485,22 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -3437,11 +3554,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "zerocopy-derive 0.8.23", + "zerocopy-derive 0.8.24", ] [[package]] @@ -3457,9 +3574,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index fa79b82f..f8dbca49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,5 +32,6 @@ strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -# [patch."https://github.com/stackabletech/operator-rs.git"] +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { path = "../operator-rs/crates/stackable-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d87cef34..b4b655d8 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -94,7 +94,7 @@ use crate::{ kerberos_container_start_commands, }, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}, + product_logging::extend_role_group_config_map, }; pub const HIVE_CONTROLLER_NAME: &str = "hivecluster"; @@ -228,10 +228,8 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to resolve the Vector aggregator address"))] - ResolveVectorAggregatorAddress { - source: crate::product_logging::Error, - }, + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] InvalidLoggingConfig { @@ -439,10 +437,6 @@ pub async fn reconcile_hive( .await .context(ApplyRoleServiceSnafu)?; - let vector_aggregator_address = resolve_vector_aggregator_address(hive, client) - .await - .context(ResolveVectorAggregatorAddressSnafu)?; - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); for (rolegroup_name, rolegroup_config) in metastore_config.iter() { @@ -461,7 +455,6 @@ pub async fn reconcile_hive( rolegroup_config, s3_connection_spec.as_ref(), &config, - vector_aggregator_address.as_deref(), &client.kubernetes_cluster_info, )?; let rg_statefulset = build_metastore_rolegroup_statefulset( @@ -604,7 +597,6 @@ fn build_metastore_rolegroup_config_map( role_group_config: &HashMap>, s3_connection_spec: Option<&S3ConnectionSpec>, merged_config: &MetaStoreConfig, - vector_aggregator_address: Option<&str>, cluster_info: &KubernetesClusterInfo, ) -> Result { let mut hive_site_data = String::new(); @@ -720,7 +712,6 @@ fn build_metastore_rolegroup_config_map( extend_role_group_config_map( rolegroup, - vector_aggregator_address, &merged_config.logging, &mut cm_builder, ) @@ -1049,6 +1040,13 @@ fn build_metastore_rolegroup_statefulset( // N.B. the vector container should *follow* the hive container so that the hive one is the // default, is started first and can provide any dependencies that vector expects if merged_config.logging.enable_vector_agent { + match hive + .spec + .cluster_config + .vector_aggregator_config_map_name + .to_owned() + { + Some(vector_aggregator_config_map_name) => { pod_builder.add_container( product_logging::framework::vector_container( resolved_product_image, @@ -1061,10 +1059,16 @@ fn build_metastore_rolegroup_statefulset( .with_memory_request("128Mi") .with_memory_limit("128Mi") .build(), + &vector_aggregator_config_map_name, ) .context(BuildVectorContainerSnafu)?, ); } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } +} + } let mut pod_template = pod_builder.build_template(); pod_template.merge_from(role.config.pod_overrides.clone()); diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 9e6da94f..750cb472 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,9 +1,6 @@ -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::Snafu; use stackable_operator::{ builder::configmap::ConfigMapBuilder, - client::Client, - k8s_openapi::api::core::v1::ConfigMap, - kube::ResourceExt, memory::BinaryMultiple, product_logging::{ self, @@ -33,54 +30,16 @@ pub enum Error { }, #[snafu(display("crd validation failure"))] CrdValidationFailure { source: crate::crd::Error }, - #[snafu(display("vectorAggregatorConfigMapName must be set"))] - MissingVectorAggregatorAddress, } type Result = std::result::Result; -const VECTOR_AGGREGATOR_CM_ENTRY: &str = "ADDRESS"; const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %5p [%t] %c{2}: %m%n"; const HIVE_LOG_FILE: &str = "hive.log4j2.xml"; -/// Return the address of the Vector aggregator if the corresponding ConfigMap name is given in the -/// cluster spec -pub async fn resolve_vector_aggregator_address( - hive: &v1alpha1::HiveCluster, - client: &Client, -) -> Result> { - let vector_aggregator_address = if let Some(vector_aggregator_config_map_name) = - &hive.spec.cluster_config.vector_aggregator_config_map_name - { - let vector_aggregator_address = client - .get::( - vector_aggregator_config_map_name, - hive.namespace() - .as_deref() - .context(ObjectHasNoNamespaceSnafu)?, - ) - .await - .context(ConfigMapNotFoundSnafu { - cm_name: vector_aggregator_config_map_name.to_string(), - })? - .data - .and_then(|mut data| data.remove(VECTOR_AGGREGATOR_CM_ENTRY)) - .context(MissingConfigMapEntrySnafu { - entry: VECTOR_AGGREGATOR_CM_ENTRY, - cm_name: vector_aggregator_config_map_name.to_string(), - })?; - Some(vector_aggregator_address) - } else { - None - }; - - Ok(vector_aggregator_address) -} - /// Extend the role group ConfigMap with logging and Vector configurations pub fn extend_role_group_config_map( rolegroup: &RoleGroupRef, - vector_aggregator_address: Option<&str>, logging: &Logging, cm_builder: &mut ConfigMapBuilder, ) -> Result<()> { @@ -120,7 +79,6 @@ pub fn extend_role_group_config_map( product_logging::framework::VECTOR_CONFIG_FILE, product_logging::framework::create_vector_config( rolegroup, - vector_aggregator_address.context(MissingVectorAggregatorAddressSnafu)?, vector_log_config, ), ); From 5bfb3cdcf4c3ecf1f8dcbccebd577258c5a89920 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 7 Apr 2025 21:16:55 +0200 Subject: [PATCH 3/7] add changelog entry --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a557903..7986f5f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Inject the vector aggregator address into the vector config using the env var `VECTOR_AGGREGATOR_ADDRESS` ([#589]). + +### Fixed + +- Fix a bug where changes to ConfigMaps that are referenced in the HiveCluster spec didn't trigger a reconciliation ([#589]). + +[#589]: https://github.com/stackabletech/hdfs-operator/pull/589 + ## [25.3.0] - 2025-03-21 ### Added From d916da205708b40b26002d442645d2446e62f9ae Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 8 Apr 2025 12:03:21 +0200 Subject: [PATCH 4/7] remove local patch for operator-rs --- Cargo.lock | 3 +++ Cargo.toml | 6 +++--- rust/operator-binary/src/main.rs | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0457df9..9e889441 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2671,6 +2671,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.90.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#ea063b4595caa20c82d37c595487c76476c9ab10" dependencies = [ "chrono", "clap", @@ -2705,6 +2706,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#ea063b4595caa20c82d37c595487c76476c9ab10" dependencies = [ "darling", "proc-macro2", @@ -2715,6 +2717,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#ea063b4595caa20c82d37c595487c76476c9ab10" dependencies = [ "kube", "semver", diff --git a/Cargo.toml b/Cargo.toml index 5079dccb..7d5e23d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/hive-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.89.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.90.0" } stackable-telemetry = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-telemetry-0.4.0" } stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.7.1" } @@ -33,6 +33,6 @@ strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -[patch."https://github.com/stackabletech/operator-rs.git"] -stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +# [patch."https://github.com/stackabletech/operator-rs.git"] +# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index ed44634c..f0a5ad0f 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -22,7 +22,7 @@ use stackable_operator::{ runtime::{ events::{Recorder, Reporter}, reflector::ObjectRef, watcher, Controller }, ResourceExt, - }, logging::controller::report_controller_reconciled, shared::yaml::SerializeOptions, YamlSchema + }, logging::controller::report_controller_reconciled, shared::yaml::SerializeOptions, }; use stackable_telemetry::{Tracing, tracing::settings::Settings}; use tracing::level_filters::LevelFilter; From e5222bb5579d93b1e4769f10ec644cca0939b73c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 8 Apr 2025 12:11:16 +0200 Subject: [PATCH 5/7] run cargo fmt --- rust/operator-binary/src/controller.rs | 67 +++++----- rust/operator-binary/src/crd/affinity.rs | 65 +++++----- rust/operator-binary/src/main.rs | 131 +++++++++++--------- rust/operator-binary/src/product_logging.rs | 5 +- 4 files changed, 138 insertions(+), 130 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b4b655d8..4cb484f3 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -710,14 +710,11 @@ fn build_metastore_rolegroup_config_map( cm_builder.add_data(CORE_SITE_XML, to_hadoop_xml(data.iter())); } - extend_role_group_config_map( - rolegroup, - &merged_config.logging, - &mut cm_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; + extend_role_group_config_map(rolegroup, &merged_config.logging, &mut cm_builder).context( + InvalidLoggingConfigSnafu { + cm_name: rolegroup.object_name(), + }, + )?; cm_builder .build() @@ -1041,33 +1038,33 @@ fn build_metastore_rolegroup_statefulset( // default, is started first and can provide any dependencies that vector expects if merged_config.logging.enable_vector_agent { match hive - .spec - .cluster_config - .vector_aggregator_config_map_name - .to_owned() - { - Some(vector_aggregator_config_map_name) => { - pod_builder.add_container( - product_logging::framework::vector_container( - resolved_product_image, - STACKABLE_CONFIG_MOUNT_DIR_NAME, - STACKABLE_LOG_DIR_NAME, - merged_config.logging.containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - &vector_aggregator_config_map_name, - ) - .context(BuildVectorContainerSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } -} + .spec + .cluster_config + .vector_aggregator_config_map_name + .to_owned() + { + Some(vector_aggregator_config_map_name) => { + pod_builder.add_container( + product_logging::framework::vector_container( + resolved_product_image, + STACKABLE_CONFIG_MOUNT_DIR_NAME, + STACKABLE_LOG_DIR_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + &vector_aggregator_config_map_name, + ) + .context(BuildVectorContainerSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } } let mut pod_template = pod_builder.build_template(); diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 980816a7..1cde42e1 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -61,36 +61,39 @@ mod tests { .merged_config(&role, &role.rolegroup_ref(&hive, "default")) .unwrap(); - assert_eq!(merged_config.affinity, StackableAffinity { - pod_affinity: None, - pod_anti_affinity: Some(PodAntiAffinity { - preferred_during_scheduling_ignored_during_execution: Some(vec![ - WeightedPodAffinityTerm { - pod_affinity_term: PodAffinityTerm { - label_selector: Some(LabelSelector { - match_labels: Some(BTreeMap::from([ - ("app.kubernetes.io/name".to_string(), "hive".to_string(),), - ( - "app.kubernetes.io/instance".to_string(), - "simple-hive".to_string(), - ), - ( - "app.kubernetes.io/component".to_string(), - "metastore".to_string(), - ) - ])), - ..LabelSelector::default() - }), - topology_key: "kubernetes.io/hostname".to_string(), - ..PodAffinityTerm::default() - }, - weight: 70 - } - ]), - required_during_scheduling_ignored_during_execution: None, - }), - node_affinity: None, - node_selector: None, - }); + assert_eq!( + merged_config.affinity, + StackableAffinity { + pod_affinity: None, + pod_anti_affinity: Some(PodAntiAffinity { + preferred_during_scheduling_ignored_during_execution: Some(vec![ + WeightedPodAffinityTerm { + pod_affinity_term: PodAffinityTerm { + label_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), "hive".to_string(),), + ( + "app.kubernetes.io/instance".to_string(), + "simple-hive".to_string(), + ), + ( + "app.kubernetes.io/component".to_string(), + "metastore".to_string(), + ) + ])), + ..LabelSelector::default() + }), + topology_key: "kubernetes.io/hostname".to_string(), + ..PodAffinityTerm::default() + }, + weight: 70 + } + ]), + required_during_scheduling_ignored_during_execution: None, + }), + node_affinity: None, + node_selector: None, + } + ); } } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index f0a5ad0f..4ef1a363 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -17,12 +17,19 @@ use stackable_operator::{ k8s_openapi::api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, Service}, - }, kube::{ + }, + kube::{ + ResourceExt, core::DeserializeGuard, runtime::{ - events::{Recorder, Reporter}, reflector::ObjectRef, watcher, Controller - }, ResourceExt, - }, logging::controller::report_controller_reconciled, shared::yaml::SerializeOptions, + Controller, + events::{Recorder, Reporter}, + reflector::ObjectRef, + watcher, + }, + }, + logging::controller::report_controller_reconciled, + shared::yaml::SerializeOptions, }; use stackable_telemetry::{Tracing, tracing::settings::Settings}; use tracing::level_filters::LevelFilter; @@ -118,66 +125,70 @@ async fn main() -> anyhow::Result<()> { &cluster_info_opts, ) .await?; - let event_recorder = Arc::new(Recorder::new(client.as_kube_client(), Reporter { - controller: HIVE_FULL_CONTROLLER_NAME.to_string(), - instance: None, - })); + let event_recorder = Arc::new(Recorder::new( + client.as_kube_client(), + Reporter { + controller: HIVE_FULL_CONTROLLER_NAME.to_string(), + instance: None, + }, + )); let hive_controller = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default(), ); let hive_store_1 = hive_controller.store(); - hive_controller.owns( - watch_namespace.get_api::(&client), - watcher::Config::default(), - ) - .owns( - watch_namespace.get_api::(&client), - watcher::Config::default(), - ) - .owns( - watch_namespace.get_api::(&client), - watcher::Config::default(), - ) - .shutdown_on_signal() - .watches( - watch_namespace.get_api::>(&client), - watcher::Config::default(), - move |config_map| { - hive_store_1 - .state() - .into_iter() - .filter(move |hive| references_config_map(hive, &config_map)) - .map(|hive| ObjectRef::from_obj(&*hive)) - }, - ) - .run( - controller::reconcile_hive, - controller::error_policy, - Arc::new(controller::Ctx { - client: client.clone(), - product_config, - }), - ) - // We can let the reporting happen in the background - .for_each_concurrent( - 16, // concurrency limit - |result| { - // The event_recorder needs to be shared across all invocations, so that - // events are correctly aggregated - let event_recorder = event_recorder.clone(); - async move { - report_controller_reconciled( - &event_recorder, - HIVE_FULL_CONTROLLER_NAME, - &result, - ) - .await; - } - }, - ) - .await; + hive_controller + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) + .shutdown_on_signal() + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + move |config_map| { + hive_store_1 + .state() + .into_iter() + .filter(move |hive| references_config_map(hive, &config_map)) + .map(|hive| ObjectRef::from_obj(&*hive)) + }, + ) + .run( + controller::reconcile_hive, + controller::error_policy, + Arc::new(controller::Ctx { + client: client.clone(), + product_config, + }), + ) + // We can let the reporting happen in the background + .for_each_concurrent( + 16, // concurrency limit + |result| { + // The event_recorder needs to be shared across all invocations, so that + // events are correctly aggregated + let event_recorder = event_recorder.clone(); + async move { + report_controller_reconciled( + &event_recorder, + HIVE_FULL_CONTROLLER_NAME, + &result, + ) + .await; + } + }, + ) + .await; } } @@ -195,5 +206,5 @@ fn references_config_map( match hive.spec.cluster_config.hdfs.to_owned() { Some(hdfs_connection) => hdfs_connection.config_map == config_map.name_any(), None => false, - } -} \ No newline at end of file + } +} diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 750cb472..30e9318b 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -77,10 +77,7 @@ pub fn extend_role_group_config_map( if logging.enable_vector_agent { cm_builder.add_data( product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config( - rolegroup, - vector_log_config, - ), + product_logging::framework::create_vector_config(rolegroup, vector_log_config), ); } From d313488c6abbde0c6223346cab395cb0247d2808 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 9 Apr 2025 16:10:40 +0200 Subject: [PATCH 6/7] rename store --- rust/operator-binary/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index aa710422..7ccc7398 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -134,7 +134,7 @@ async fn main() -> anyhow::Result<()> { watch_namespace.get_api::>(&client), watcher::Config::default(), ); - let hive_store_1 = hive_controller.store(); + let config_map_store = hive_controller.store(); hive_controller .owns( watch_namespace.get_api::(&client), @@ -153,7 +153,7 @@ async fn main() -> anyhow::Result<()> { watch_namespace.get_api::>(&client), watcher::Config::default(), move |config_map| { - hive_store_1 + config_map_store .state() .into_iter() .filter(move |hive| references_config_map(hive, &config_map)) From c10582bdb08596c7b32759408954dd973fd2d2fa Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 9 Apr 2025 18:44:50 +0200 Subject: [PATCH 7/7] chore: Use borrows --- rust/operator-binary/src/controller.rs | 9 ++------- rust/operator-binary/src/main.rs | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4cb484f3..d6f9160c 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1037,12 +1037,7 @@ fn build_metastore_rolegroup_statefulset( // N.B. the vector container should *follow* the hive container so that the hive one is the // default, is started first and can provide any dependencies that vector expects if merged_config.logging.enable_vector_agent { - match hive - .spec - .cluster_config - .vector_aggregator_config_map_name - .to_owned() - { + match &hive.spec.cluster_config.vector_aggregator_config_map_name { Some(vector_aggregator_config_map_name) => { pod_builder.add_container( product_logging::framework::vector_container( @@ -1056,7 +1051,7 @@ fn build_metastore_rolegroup_statefulset( .with_memory_request("128Mi") .with_memory_limit("128Mi") .build(), - &vector_aggregator_config_map_name, + vector_aggregator_config_map_name, ) .context(BuildVectorContainerSnafu)?, ); diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 7ccc7398..c45342db 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -200,7 +200,7 @@ fn references_config_map( return false; }; - match hive.spec.cluster_config.hdfs.to_owned() { + match &hive.spec.cluster_config.hdfs { Some(hdfs_connection) => hdfs_connection.config_map == config_map.name_any(), None => false, }