From b84be08e40fec32b401c499b3dd7b4bd5636f5cb Mon Sep 17 00:00:00 2001 From: maltesander Date: Fri, 5 Mar 2021 16:15:11 +0100 Subject: [PATCH 01/10] Adapted to operator-rs changes --- operator/src/lib.rs | 3 ++- server/src/main.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/operator/src/lib.rs b/operator/src/lib.rs index 0a315e2..4766382 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -15,12 +15,13 @@ use async_trait::async_trait; use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; use stackable_operator::client::Client; use stackable_operator::conditions::ConditionStatus; +use stackable_operator::config_map::create_config_map; use stackable_operator::controller::{Controller, ControllerStrategy, ReconciliationState}; use stackable_operator::error::OperatorResult; +use stackable_operator::krustlet::create_tolerations; use stackable_operator::reconcile::{ ReconcileFunctionAction, ReconcileResult, ReconciliationContext, }; -use stackable_operator::{create_config_map, create_tolerations}; use stackable_operator::{finalizer, metadata, podutils}; use stackable_spark_crd::{ SparkCluster, SparkClusterSpec, SparkClusterStatus, SparkNodeSelector, SparkNodeType, diff --git a/server/src/main.rs b/server/src/main.rs index 0e25c50..16c5370 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -3,7 +3,7 @@ use stackable_spark_crd::SparkCluster; #[tokio::main] async fn main() -> Result<(), error::Error> { - stackable_operator::initialize_logging("SPARK_OPERATOR_LOG"); + stackable_operator::logging::initialize_logging("SPARK_OPERATOR_LOG"); let client = client::create_client(Some("spark.stackable.tech".to_string())).await?; stackable_operator::crd::ensure_crd_created::(client.clone()).await?; From f3295347fc19e32979c6e4210b326a1662c7c594 Mon Sep 17 00:00:00 2001 From: maltesander Date: Fri, 5 Mar 2021 16:16:45 +0100 Subject: [PATCH 02/10] Updated and working with operator-rs (https://github.com/stackabletech/operator-rs/pull/77) --- Cargo.lock | 1901 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1901 insertions(+) create mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..ff3d63c --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1901 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "array_tool" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271" + +[[package]] +name = "async-trait" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4c64e223db1fffa7683a719921434caa880463cfa5820032b063c9ecd5cc49" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bumpalo" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time", + "winapi", +] + +[[package]] +name = "const_format" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0412fd9e3c921f868af82a0097da41c250087e513786858b9e6b6055f8ed300" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df496e1bbc93814d728a8036ff054cd95830afe9cf2275c9326688c02eff936" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if", + "num_cpus", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "dtoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d7ed2934d741c6b37e33e3832298e8850b53fd2d2bea03873375596c7cea4e" + +[[package]] +name = "dyn-clone" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" + +[[package]] +name = "futures-executor" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" + +[[package]] +name = "futures-macro" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" + +[[package]] +name = "futures-task" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" + +[[package]] +name = "futures-util" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hyper" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.5", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "jsonpath_lib" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61352ec23883402b7d30b3313c16cbabefb8907361c4eb669d990cbb87ceee5a" +dependencies = [ + "array_tool", + "env_logger", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "k8s-openapi" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc1f973542059e6d5a6d63de6a9539d0ec784f82b2327f3c1915d33200bc6a4" +dependencies = [ + "base64", + "bytes", + "chrono", + "serde", + "serde-value", + "serde_json", +] + +[[package]] +name = "kube" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "362fa64584999b25670b73f4874d716c14b039351b2da6740c7610a3d2e970cf" +dependencies = [ + "Inflector", + "base64", + "bytes", + "chrono", + "dirs-next", + "either", + "futures", + "http", + "hyper", + "hyper-rustls", + "hyper-timeout", + "hyper-tls", + "jsonpath_lib", + "k8s-openapi", + "kube-derive", + "log", + "openssl", + "pem", + "pin-project 1.0.5", + "serde", + "serde_json", + "serde_yaml", + "static_assertions", + "thiserror", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tokio-util", + "tower", + "url", +] + +[[package]] +name = "kube-derive" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b58ffd8a1bd162d64ac423d7c1ea1de70a4c743a9d18bc1bf364f293fcd184a" +dependencies = [ + "Inflector", + "proc-macro2", + "quote", + "serde_json", + "syn", +] + +[[package]] +name = "kube-runtime" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b82772bce36d85f6825d7db915e117c5bbf1cc13e8c95cd91281a55ec7cfd9bb" +dependencies = [ + "dashmap", + "derivative", + "futures", + "k8s-openapi", + "kube", + "pin-project 1.0.5", + "serde", + "smallvec", + "snafu", + "tokio", + "tokio-util", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265d751d31d6780a3f956bb5b8022feba2d94eeee5a84ba64f4212eedca42213" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "mio" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi", +] + +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" + +[[package]] +name = "openssl" +version = "0.10.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" +dependencies = [ + "num-traits", +] + +[[package]] +name = "pem" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" +dependencies = [ + "base64", + "once_cell", + "regex", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" +dependencies = [ + "pin-project-internal 1.0.5", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "product-config" +version = "0.1.0" +source = "git+https://github.com/stackabletech/product-config.git?branch=start#2c3642d193431e306ddb5e6bb9e82106536e664c" +dependencies = [ + "regex", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom", + "redox_syscall", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "schemars" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763f667253711994847f7e73befe859d6fff7bea2b7a7f01669d2c5b60765c37" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1d457e2e37415f32b7628ddc5a7fea06ef63bd029ed180d65166e87ca25ce21" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfd318104249865096c8da1dfabf09ddbb6d0330ea176812a62ec75e40c4166" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dbab34ca63057a1f15280bdf3c39f2b1eb1b54c17e98360e511637aef7418c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15654ed4ab61726bf918a39cb8d98a2e2995b002387807fa6ba58fdf7f59bb23" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + +[[package]] +name = "sharded-slab" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "snafu" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab12d3c261b2308b0d80c26fffb58d17eba81a4be97890101f416b478c79ca7" +dependencies = [ + "doc-comment", + "futures-core", + "pin-project 0.4.27", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1508efa03c362e23817f96cde18abed596a25219a8b2c66e8db33c03543d315b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if", + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "stackable-operator" +version = "0.1.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=main#b0ffaefe3d4ac4c4478aaf3d8aec6c61a33a961d" +dependencies = [ + "async-trait", + "chrono", + "const_format", + "either", + "futures", + "k8s-openapi", + "kube", + "kube-runtime", + "lazy_static", + "regex", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio", + "tracing", + "tracing-futures", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "stackable-spark-crd" +version = "0.1.0" +dependencies = [ + "derivative", + "k8s-openapi", + "kube", + "schemars", + "semver", + "serde", + "serde_json", + "stackable-operator", + "strum", + "strum_macros", + "thiserror", +] + +[[package]] +name = "stackable-spark-operator" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures", + "k8s-openapi", + "kube", + "kube-runtime", + "product-config", + "semver", + "serde", + "serde_json", + "stackable-operator", + "stackable-spark-crd", + "thiserror", + "tracing", + "uuid", +] + +[[package]] +name = "stackable-spark-operator-server" +version = "0.1.0-nightly" +dependencies = [ + "stackable-operator", + "stackable-spark-crd", + "stackable-spark-operator", + "tokio", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7318c509b5ba57f18533982607f24070a55d353e90d4cae30c467cdb2ad5ac5c" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed22b90a0e734a23a7610f4283ac9e5acfb96cbb30dfefa540d66f866f1c09c5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +dependencies = [ + "autocfg", + "libc", + "mio", + "num_cpus", + "once_cell", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90c49f106be240de154571dd31fbe48acb10ba6c6dd6f6517ad603abffa42de9" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tower" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f715efe02c0862926eb463e49368d38ddb119383475686178e32e26d15d06a66" +dependencies = [ + "futures-core", + "futures-util", + "pin-project 1.0.5", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a9bd1db7706f2373a190b0d067146caa39350c486f3d455b0e33b431f94c07" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project 1.0.5", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab8966ac3ca27126141f7999361cc97dd6fb4b71da04c02044fa9045d98bb96" +dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" + +[[package]] +name = "web-sys" +version = "0.3.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] From 4404d103df22163299e589059e84d335965c8b6f Mon Sep 17 00:00:00 2001 From: maltesander Date: Sun, 7 Mar 2021 19:33:22 +0100 Subject: [PATCH 03/10] Added pod_utils for pod related methods. Adjusted get_master_urls in preparation to add to pod labels to keep track in worker pods which masters are known (in case masters change / scale / delete etc.) --- crd/src/lib.rs | 6 +- operator/src/config.rs | 122 ++++++++++++--------------------- operator/src/lib.rs | 151 ++++++----------------------------------- 3 files changed, 66 insertions(+), 213 deletions(-) diff --git a/crd/src/lib.rs b/crd/src/lib.rs index 163f28c..0f3574c 100644 --- a/crd/src/lib.rs +++ b/crd/src/lib.rs @@ -173,9 +173,9 @@ impl SparkNodeType { /// Returns the container start command for a spark node /// Right now works only for images using hadoop2.7 /// # Arguments - /// * `version` - current specified SparkVersion + /// * `version` - Current specified cluster version /// - pub fn get_command(&self, version: &SparkVersion) -> String { + pub fn get_command(&self, version: &str) -> String { // TODO: remove hardcoded and adapt for versioning format!( "spark-{}-bin-hadoop2.7/sbin/start-{}.sh", @@ -263,14 +263,12 @@ impl SparkVersion { pub fn is_upgrade(&self, to: &Self) -> Result { let from_version = Version::parse(&self.to_string())?; let to_version = Version::parse(&to.to_string())?; - Ok(to_version > from_version) } pub fn is_downgrade(&self, to: &Self) -> Result { let from_version = Version::parse(&self.to_string())?; let to_version = Version::parse(&to.to_string())?; - Ok(to_version < from_version) } } diff --git a/operator/src/config.rs b/operator/src/config.rs index 61fc496..8e30252 100644 --- a/operator/src/config.rs +++ b/operator/src/config.rs @@ -1,4 +1,4 @@ -use k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EnvVar, Volume, VolumeMount}; +use k8s_openapi::api::core::v1::EnvVar; use stackable_spark_crd::{ ConfigOption, SparkClusterSpec, SparkNode, SparkNodeSelector, SparkNodeType, }; @@ -31,124 +31,90 @@ const SPARK_HISTORY_UI_PORT: &str = "spark.history.ui.port"; /// The worker start command needs to be extended with all known master nodes and ports. /// The required URLs are in format: 'spark://: Option { - let mut master_url: String = String::new(); +pub fn adapt_worker_command(node_type: &SparkNodeType, master: &SparkNode) -> Option { + let mut adapted_command: String = String::new(); // only for workers if node_type != &SparkNodeType::Worker { return None; } + + let master_urls = get_master_urls(master); + for url in master_urls { + if !adapted_command.is_empty() { + adapted_command.push(','); + } + adapted_command.push_str(url.as_str()); + } + + Some(adapted_command) +} + +/// The master port can be configured and needs to be checked in config / env or general options. +/// Defaults to 7077 if no port is specified. +/// +/// # Arguments +/// * `master` - Master SparkNode containing the required settings +/// +pub fn get_master_urls(master: &SparkNode) -> Vec { + let mut master_urls = vec![]; // get all available master selectors for selector in &master.selectors { // check in conf properties and env variables for port // conf properties have higher priority than env variables if let Some(conf) = &selector.config { - if let Some(master) = - search_master_port(&selector.node_name, SPARK_MASTER_PORT_CONF, conf) - { - master_url.push_str(master.as_str()); + if let Some(port) = search_master_port(SPARK_MASTER_PORT_CONF, conf) { + master_urls.push(format!( + "{}{}:{}", + SPARK_URL_START, selector.node_name, port + )); continue; } } else if let Some(env) = &selector.env { - if let Some(master) = - search_master_port(&selector.node_name, SPARK_MASTER_PORT_ENV, env) - { - master_url.push_str(master.as_str()); + if let Some(port) = search_master_port(SPARK_MASTER_PORT_ENV, env) { + master_urls.push(format!( + "{}{}:{}", + SPARK_URL_START, selector.node_name, port + )); continue; } } else if let Some(port) = selector.master_port { - master_url - .push_str(format!("{}{}:{},", SPARK_URL_START, selector.node_name, port).as_str()); + master_urls.push(format!( + "{}{}:{}", + SPARK_URL_START, selector.node_name, port + )); continue; } // TODO: default to default value in product conf - master_url - .push_str(format!("{}{}:{},", SPARK_URL_START, selector.node_name, "7077").as_str()); + master_urls.push(format!( + "{}{}:{}", + SPARK_URL_START, selector.node_name, "7077" + )); } - Some(master_url) + master_urls } /// Search for a master port in config properties or env variables /// /// # Arguments -/// * `node_name` - Node IP / DNS address /// * `option_name` - Name of the option to look for e.g. "SPARK_MASTER_PORT" /// * `options` - Vec of config properties or env variables /// -fn search_master_port( - node_name: &str, - option_name: &str, - options: &[ConfigOption], -) -> Option { +fn search_master_port(option_name: &str, options: &[ConfigOption]) -> Option { for option in options { if option.name == option_name { - return Some(format!( - "{}{}:{},", - SPARK_URL_START, node_name, option.value - )); + return Some(option.value.clone()); } } None } -const CONFIG_VOLUME: &str = "config-volume"; -const EVENT_VOLUME: &str = "event-volume"; - -/// Create volume mounts for the spark config files and optional an event dir for spark logs -/// -/// # Arguments -/// * `log_dir` - Event/Log dir for SparkNodes. History Server reads these logs to offer metrics -/// -pub fn create_volume_mounts(log_dir: &Option) -> Vec { - let mut volume_mounts = vec![VolumeMount { - mount_path: "conf".to_string(), - name: CONFIG_VOLUME.to_string(), - ..VolumeMount::default() - }]; - // if log dir is provided, create another folder for logDir - if let Some(dir) = log_dir { - volume_mounts.push(VolumeMount { - mount_path: dir.clone(), - name: EVENT_VOLUME.to_string(), - ..VolumeMount::default() - }); - } - - volume_mounts -} - -/// Create a volume to store the spark config files and optional an event volume for spark logs -/// -/// # Arguments -/// * `configmap_name` - ConfigMap name where the required spark configuration files (spark-defaults.conf and spark-env.sh) are located -/// -pub fn create_volumes(configmap_name: &str) -> Vec { - let volumes = vec![ - Volume { - name: CONFIG_VOLUME.to_string(), - config_map: Some(ConfigMapVolumeSource { - name: Some(configmap_name.to_string()), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }, - Volume { - name: EVENT_VOLUME.to_string(), - ..Volume::default() - }, - ]; - - volumes -} - /// The SPARK_CONFIG_DIR and SPARK_NO_DAEMONIZE must be provided as env variable in the container. /// SPARK_CONFIG_DIR must be available before the start up of the nodes (master, worker, history-server) to point to our custom configuration. /// SPARK_NO_DAEMONIZE stops the node processes to be started in the background, which causes the agent to lose track of the processes. diff --git a/operator/src/lib.rs b/operator/src/lib.rs index 4766382..a34b29e 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -1,13 +1,14 @@ #![feature(backtrace)] mod config; mod error; +mod pod_utils; use crate::error::Error; use kube::Api; use tracing::{debug, error, info, trace}; -use k8s_openapi::api::core::v1::{ConfigMap, Container, Pod, PodSpec, Volume}; +use k8s_openapi::api::core::v1::{ConfigMap, Pod}; use kube::api::{ListParams, Meta}; use serde_json::json; @@ -18,11 +19,10 @@ use stackable_operator::conditions::ConditionStatus; use stackable_operator::config_map::create_config_map; use stackable_operator::controller::{Controller, ControllerStrategy, ReconciliationState}; use stackable_operator::error::OperatorResult; -use stackable_operator::krustlet::create_tolerations; use stackable_operator::reconcile::{ ReconcileFunctionAction, ReconcileResult, ReconciliationContext, }; -use stackable_operator::{finalizer, metadata, podutils}; +use stackable_operator::{finalizer, podutils}; use stackable_spark_crd::{ SparkCluster, SparkClusterSpec, SparkClusterStatus, SparkNodeSelector, SparkNodeType, SparkVersion, @@ -32,17 +32,9 @@ use std::future::Future; use std::pin::Pin; use std::str::FromStr; use std::time::Duration; -use uuid::Uuid; const FINALIZER_NAME: &str = "spark.stackable.tech/cleanup"; -/// Pod label which holds the selector hash in the pods to identify which selector they belong to -const HASH_LABEL: &str = "spark.stackable.tech/hash"; -/// Pod label which holds node role / type (master, worker, history-server) in the pods -const TYPE_LABEL: &str = "spark.stackable.tech/type"; -/// Pod label which indicates the cluster version it was created for -const VERSION_LABEL: &str = "spark.stackable.tech/currentVersion"; - type SparkReconcileResult = ReconcileResult; struct SparkState { @@ -346,9 +338,9 @@ impl SparkState { if let Some(labels) = pod.metadata.labels.clone() { // we require HASH and TYPE label to identify and sort the pods into the NodeInformation if let (Some(hash), Some(node_type), Some(version)) = ( - labels.get(HASH_LABEL), - labels.get(TYPE_LABEL), - labels.get(VERSION_LABEL), + labels.get(pod_utils::HASH_LABEL), + labels.get(pod_utils::TYPE_LABEL), + labels.get(pod_utils::VERSION_LABEL), ) { let spark_node_type = match SparkNodeType::from_str(node_type) { Ok(nt) => nt, @@ -356,7 +348,7 @@ impl SparkState { error!( "Pod [{}] has an invalid type '{}' [{}], deleting it.", Meta::name(&pod), - TYPE_LABEL, + pod_utils::TYPE_LABEL, node_type ); self.context.client.delete(&pod).await?; @@ -370,7 +362,7 @@ impl SparkState { error!( "Pod [{}] has an outdated '{}' [{}], deleting it.", Meta::name(&pod), - HASH_LABEL, + pod_utils::HASH_LABEL, hash ); self.context.client.delete(&pod).await?; @@ -386,7 +378,7 @@ impl SparkState { info!( "Pod [{}] has an outdated '{}' [{}] - required is [{}], deleting it", Meta::name(&pod), - VERSION_LABEL, + pod_utils::VERSION_LABEL, version, target_version ); @@ -410,7 +402,7 @@ impl SparkState { } else { // some labels missing error!("Pod [{}] is missing one or more required '{:?}' labels, this is illegal, deleting it.", - Meta::name(&pod), vec![HASH_LABEL, TYPE_LABEL]); + Meta::name(&pod), vec![pod_utils::HASH_LABEL, pod_utils::TYPE_LABEL, pod_utils::VERSION_LABEL]); self.context.client.delete(&pod).await?; } } else { @@ -651,79 +643,18 @@ impl SparkState { node_type: &SparkNodeType, hash: &str, ) -> Result { - let pod = self.build_pod(selector, hash, node_type)?; + let pod = pod_utils::build_pod( + &self.context, + node_type, + selector, + &self.spec.master, + hash, + &self.spec.version.to_string(), + &self.spec.log_dir, + )?; Ok(self.context.client.create(&pod).await?) } - /// Build a pod using its selector and node_type - /// - /// # Arguments - /// * `selector` - SparkNodeSelector which contains specific pod information - /// * `node_type` - SparkNodeType (master/worker/history-server) - /// * `hash` - NodeSelector hash - /// - fn build_pod( - &self, - selector: &SparkNodeSelector, - hash: &str, - node_type: &SparkNodeType, - ) -> Result { - let (containers, volumes) = self.build_containers(node_type, hash); - - Ok(Pod { - metadata: metadata::build_metadata( - self.create_pod_name(node_type, hash), - Some(self.build_labels(node_type, hash)), - &self.context.resource, - true, - )?, - spec: Some(PodSpec { - node_name: Some(selector.node_name.clone()), - tolerations: Some(create_tolerations()), - containers, - volumes: Some(volumes), - ..PodSpec::default() - }), - ..Pod::default() - }) - } - - /// Build required pod containers - /// - /// # Arguments - /// * `node_type` - SparkNodeType (master/worker/history-server) - /// * `hash` - NodeSelector hash - /// - fn build_containers( - &self, - node_type: &SparkNodeType, - hash: &str, - ) -> (Vec, Vec) { - let image_name = format!("spark:{}", &self.spec.version); - - // adapt worker command with master url(s) - let mut command = vec![node_type.get_command(&self.spec.version)]; - - if let Some(adapted_command) = config::adapt_container_command(node_type, &self.spec.master) - { - command.push(adapted_command); - } - - let containers = vec![Container { - image: Some(image_name), - name: "spark".to_string(), - command: Some(command), - volume_mounts: Some(config::create_volume_mounts(&self.spec.log_dir)), - env: Some(config::create_required_startup_env()), - ..Container::default() - }]; - - let cm_name = self.create_config_map_name(node_type, hash); - let volumes = config::create_volumes(&cm_name); - - (containers, volumes) - } - /// Create required config maps for the cluster /// /// # Arguments @@ -746,54 +677,12 @@ impl SparkState { data.insert("spark-defaults.conf".to_string(), conf); data.insert("spark-env.sh".to_string(), env); - let cm_name = self.create_config_map_name(node_type, hash); + let cm_name = pod_utils::create_config_map_name(&self.context.name(), node_type, hash); let cm = create_config_map(&self.context.resource, &cm_name, data)?; self.context.client.apply_patch(&cm, &cm).await?; Ok(()) } - - /// Provide required labels for pods - /// - /// # Arguments - /// * `node_type` - SparkNodeType (master/worker/history-server) - /// * `hash` - NodeSelector hash - /// - fn build_labels(&self, node_type: &SparkNodeType, hash: &str) -> BTreeMap { - let mut labels = BTreeMap::new(); - labels.insert(TYPE_LABEL.to_string(), node_type.to_string()); - labels.insert(HASH_LABEL.to_string(), hash.to_string()); - labels.insert(VERSION_LABEL.to_string(), self.spec.version.to_string()); - - labels - } - - /// All pod names follow a simple pattern: --- - /// - /// # Arguments - /// * `node_type` - SparkNodeType (master/worker/history-server) - /// * `hash` - NodeSelector hash - /// - fn create_pod_name(&self, node_type: &SparkNodeType, hash: &str) -> String { - format!( - "{}-{}-{}-{}", - self.context.name(), - node_type.as_str(), - hash, - Uuid::new_v4().as_fields().0.to_string(), - ) - } - - /// All config map names follow a simple pattern: ---cm - /// That means multiple pods of one selector share one and the same config map - /// - /// # Arguments - /// * `node_type` - SparkNodeType (master/worker/history-server) - /// * `hash` - NodeSelector hash - /// - fn create_config_map_name(&self, node_type: &SparkNodeType, hash: &str) -> String { - format!("{}-{}-{}-cm", self.context.name(), node_type.as_str(), hash) - } } impl ReconciliationState for SparkState { From e74f303d9b11c071033ef6de1368d6ba5859c632 Mon Sep 17 00:00:00 2001 From: maltesander Date: Mon, 8 Mar 2021 09:24:28 +0100 Subject: [PATCH 04/10] Added pod_utils for pod operations --- operator/src/pod_utils.rs | 218 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 operator/src/pod_utils.rs diff --git a/operator/src/pod_utils.rs b/operator/src/pod_utils.rs new file mode 100644 index 0000000..cf3fe8a --- /dev/null +++ b/operator/src/pod_utils.rs @@ -0,0 +1,218 @@ +use crate::config; +use crate::error::Error; +use k8s_openapi::api::core::v1::{ + ConfigMapVolumeSource, Container, Pod, PodSpec, Volume, VolumeMount, +}; +use stackable_operator::krustlet::create_tolerations; +use stackable_operator::metadata; +use stackable_operator::reconcile::ReconciliationContext; +use stackable_spark_crd::{SparkCluster, SparkNode, SparkNodeSelector, SparkNodeType}; +use std::collections::hash_map::DefaultHasher; +use std::collections::BTreeMap; +use std::hash::{Hash, Hasher}; +use uuid::Uuid; + +/// Pod label which holds the selector hash in the pods to identify which selector they belong to +pub const HASH_LABEL: &str = "spark.stackable.tech/hash"; +/// Pod label which holds node role / type (master, worker, history-server) in the pods +pub const TYPE_LABEL: &str = "spark.stackable.tech/type"; +/// Pod label which indicates the cluster version it was created for +pub const VERSION_LABEL: &str = "spark.stackable.tech/currentVersion"; +/// Pod label which indicates the known master urls for a worker pod +pub const SPARK_MASTER_URLS_LABEL: &str = "spark.stackable.tech/masterUrls"; + +/// Name of the config volume to store configmap data +const CONFIG_VOLUME: &str = "config-volume"; +/// Name of the logging / event volume for SparkNode logs required by the history server +const EVENT_VOLUME: &str = "event-volume"; + +/// Build a pod using its selector and node_type +/// +/// # Arguments +/// * `context` - Reconciliation context for cluster name and resource (metadata) +/// * `node_type` - SparkNodeType (master/worker/history-server) +/// * `selector` - SparkNodeSelector which contains specific pod information +/// * `master_node` - SparkNode master to retrieve master urls for worker start commands +/// * `hash` - NodeSelector hash +/// * `version` - Current cluster version +/// * `log_dir` - Logging dir for all nodes to enable history server logs +/// +pub fn build_pod( + context: &ReconciliationContext, + node_type: &SparkNodeType, + selector: &SparkNodeSelector, + master_node: &SparkNode, + hash: &str, + version: &str, + log_dir: &Option, +) -> Result { + let cluster_name = &context.name(); + let (containers, volumes) = + build_containers(master_node, cluster_name, node_type, hash, version, log_dir); + + Ok(Pod { + metadata: metadata::build_metadata( + create_pod_name(cluster_name, node_type, hash), + Some(build_labels(node_type, hash, version, master_node)), + &context.resource, + true, + )?, + spec: Some(PodSpec { + node_name: Some(selector.node_name.clone()), + tolerations: Some(create_tolerations()), + containers, + volumes: Some(volumes), + ..PodSpec::default() + }), + ..Pod::default() + }) +} + +/// Build required pod containers +/// +/// # Arguments +/// * `master_node` - SparkNode master to retrieve master urls for worker start commands +/// * `cluster_name` - Current cluster name +/// * `node_type` - SparkNodeType (master/worker/history-server) +/// * `hash` - NodeSelector hash +/// * `version` - Current cluster version +/// * `log_dir` - Logging dir for all nodes to enable history server logs +/// +fn build_containers( + master_node: &SparkNode, + cluster_name: &str, + node_type: &SparkNodeType, + hash: &str, + version: &str, + log_dir: &Option, +) -> (Vec, Vec) { + let image_name = format!("spark:{}", version); + + let mut command = vec![node_type.get_command(version)]; + // adapt worker command with master url(s) + if let Some(master_urls) = config::adapt_worker_command(node_type, master_node) { + command.push(master_urls); + } + + let containers = vec![Container { + image: Some(image_name), + name: "spark".to_string(), + command: Some(command), + volume_mounts: Some(create_volume_mounts(log_dir)), + env: Some(config::create_required_startup_env()), + ..Container::default() + }]; + + let cm_name = create_config_map_name(cluster_name, node_type, hash); + let volumes = create_volumes(&cm_name); + + (containers, volumes) +} + +/// Create a volume to store the spark config files and optional an event volume for spark logs +/// +/// # Arguments +/// * `configmap_name` - ConfigMap name where the required spark configuration files (spark-defaults.conf and spark-env.sh) are located +/// +fn create_volumes(configmap_name: &str) -> Vec { + let volumes = vec![ + Volume { + name: CONFIG_VOLUME.to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(configmap_name.to_string()), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }, + Volume { + name: EVENT_VOLUME.to_string(), + ..Volume::default() + }, + ]; + + volumes +} + +/// Create volume mounts for the spark config files and optional an event dir for spark logs +/// +/// # Arguments +/// * `log_dir` - Event/Log dir for SparkNodes. History Server reads these logs to offer metrics +/// +pub fn create_volume_mounts(log_dir: &Option) -> Vec { + let mut volume_mounts = vec![VolumeMount { + mount_path: "conf".to_string(), + name: CONFIG_VOLUME.to_string(), + ..VolumeMount::default() + }]; + // if log dir is provided, create another folder for logDir + if let Some(dir) = log_dir { + volume_mounts.push(VolumeMount { + mount_path: dir.clone(), + name: EVENT_VOLUME.to_string(), + ..VolumeMount::default() + }); + } + + volume_mounts +} + +/// Provide required labels for pods +/// +/// # Arguments +/// * `node_type` - SparkNodeType (master/worker/history-server) +/// * `hash` - NodeSelector hash +/// * `version` - Current cluster version +/// * `master_node` - SparkNode master to retrieve master urls for worker start commands +/// +fn build_labels( + node_type: &SparkNodeType, + hash: &str, + version: &str, + master_node: &SparkNode, +) -> BTreeMap { + let mut labels = BTreeMap::new(); + labels.insert(TYPE_LABEL.to_string(), node_type.to_string()); + labels.insert(HASH_LABEL.to_string(), hash.to_string()); + labels.insert(VERSION_LABEL.to_string(), version.to_string()); + + let master_urls = config::get_master_urls(master_node); + let mut hasher = DefaultHasher::new(); + for url in master_urls { + url.hash(&mut hasher); + } + labels.insert( + SPARK_MASTER_URLS_LABEL.to_string(), + hasher.finish().to_string(), + ); + + labels +} + +/// All pod names follow a simple pattern: --- +/// +/// # Arguments +/// * `cluster_name` - Current cluster name +/// * `node_type` - SparkNodeType (master/worker/history-server) +/// * `hash` - NodeSelector hash +/// +fn create_pod_name(cluster_name: &str, node_type: &SparkNodeType, hash: &str) -> String { + format!( + "{}-{}-{}-{}", + cluster_name, + node_type.as_str(), + hash, + Uuid::new_v4().as_fields().0.to_string(), + ) +} + +/// All config map names follow a simple pattern: ---cm +/// That means multiple pods of one selector share one and the same config map +/// +/// # Arguments +/// * `cluster_name` - Current cluster name +/// * `node_type` - SparkNodeType (master/worker/history-server) +/// * `hash` - NodeSelector hash +/// +pub fn create_config_map_name(cluster_name: &str, node_type: &SparkNodeType, hash: &str) -> String { + format!("{}-{}-{}-cm", cluster_name, node_type.as_str(), hash) +} From 887539c477b9b4b883a5ef1d0682e19c736487be Mon Sep 17 00:00:00 2001 From: maltesander Date: Mon, 8 Mar 2021 10:28:37 +0100 Subject: [PATCH 05/10] Added comments, renamed parameters --- operator/src/config.rs | 8 ++++---- operator/src/lib.rs | 2 +- operator/src/pod_utils.rs | 40 ++++++++++++++++++++++++++------------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/operator/src/config.rs b/operator/src/config.rs index 8e30252..582acc7 100644 --- a/operator/src/config.rs +++ b/operator/src/config.rs @@ -58,7 +58,7 @@ pub fn adapt_worker_command(node_type: &SparkNodeType, master: &SparkNode) -> Op /// Defaults to 7077 if no port is specified. /// /// # Arguments -/// * `master` - Master SparkNode containing the required settings +/// * `master` - Master SparkNode containing the required node_name and port settings /// pub fn get_master_urls(master: &SparkNode) -> Vec { let mut master_urls = vec![]; @@ -67,7 +67,7 @@ pub fn get_master_urls(master: &SparkNode) -> Vec { // check in conf properties and env variables for port // conf properties have higher priority than env variables if let Some(conf) = &selector.config { - if let Some(port) = search_master_port(SPARK_MASTER_PORT_CONF, conf) { + if let Some(port) = get_master_port(SPARK_MASTER_PORT_CONF, conf) { master_urls.push(format!( "{}{}:{}", SPARK_URL_START, selector.node_name, port @@ -75,7 +75,7 @@ pub fn get_master_urls(master: &SparkNode) -> Vec { continue; } } else if let Some(env) = &selector.env { - if let Some(port) = search_master_port(SPARK_MASTER_PORT_ENV, env) { + if let Some(port) = get_master_port(SPARK_MASTER_PORT_ENV, env) { master_urls.push(format!( "{}{}:{}", SPARK_URL_START, selector.node_name, port @@ -106,7 +106,7 @@ pub fn get_master_urls(master: &SparkNode) -> Vec { /// * `option_name` - Name of the option to look for e.g. "SPARK_MASTER_PORT" /// * `options` - Vec of config properties or env variables /// -fn search_master_port(option_name: &str, options: &[ConfigOption]) -> Option { +fn get_master_port(option_name: &str, options: &[ConfigOption]) -> Option { for option in options { if option.name == option_name { return Some(option.value.clone()); diff --git a/operator/src/lib.rs b/operator/src/lib.rs index a34b29e..f126ab9 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -646,7 +646,7 @@ impl SparkState { let pod = pod_utils::build_pod( &self.context, node_type, - selector, + selector.node_name.as_str(), &self.spec.master, hash, &self.spec.version.to_string(), diff --git a/operator/src/pod_utils.rs b/operator/src/pod_utils.rs index cf3fe8a..0fcbfe7 100644 --- a/operator/src/pod_utils.rs +++ b/operator/src/pod_utils.rs @@ -6,7 +6,7 @@ use k8s_openapi::api::core::v1::{ use stackable_operator::krustlet::create_tolerations; use stackable_operator::metadata; use stackable_operator::reconcile::ReconciliationContext; -use stackable_spark_crd::{SparkCluster, SparkNode, SparkNodeSelector, SparkNodeType}; +use stackable_spark_crd::{SparkCluster, SparkNode, SparkNodeType}; use std::collections::hash_map::DefaultHasher; use std::collections::BTreeMap; use std::hash::{Hash, Hasher}; @@ -19,7 +19,7 @@ pub const TYPE_LABEL: &str = "spark.stackable.tech/type"; /// Pod label which indicates the cluster version it was created for pub const VERSION_LABEL: &str = "spark.stackable.tech/currentVersion"; /// Pod label which indicates the known master urls for a worker pod -pub const SPARK_MASTER_URLS_LABEL: &str = "spark.stackable.tech/masterUrls"; +pub const SPARK_MASTER_URLS_HASH_LABEL: &str = "spark.stackable.tech/masterUrls"; /// Name of the config volume to store configmap data const CONFIG_VOLUME: &str = "config-volume"; @@ -31,7 +31,7 @@ const EVENT_VOLUME: &str = "event-volume"; /// # Arguments /// * `context` - Reconciliation context for cluster name and resource (metadata) /// * `node_type` - SparkNodeType (master/worker/history-server) -/// * `selector` - SparkNodeSelector which contains specific pod information +/// * `node_name` - Specific node_name (host) of the pod /// * `master_node` - SparkNode master to retrieve master urls for worker start commands /// * `hash` - NodeSelector hash /// * `version` - Current cluster version @@ -40,7 +40,7 @@ const EVENT_VOLUME: &str = "event-volume"; pub fn build_pod( context: &ReconciliationContext, node_type: &SparkNodeType, - selector: &SparkNodeSelector, + node_name: &str, master_node: &SparkNode, hash: &str, version: &str, @@ -58,7 +58,7 @@ pub fn build_pod( true, )?, spec: Some(PodSpec { - node_name: Some(selector.node_name.clone()), + node_name: Some(node_name.to_string()), tolerations: Some(create_tolerations()), containers, volumes: Some(volumes), @@ -156,13 +156,16 @@ pub fn create_volume_mounts(log_dir: &Option) -> Vec { volume_mounts } -/// Provide required labels for pods +/// Provide required labels for pods. We need to keep track of which workers are +/// connected to which masters. This is accomplished by hashing known master urls +/// and comparing to the pods. If the hash from pod and selector differ, that means +/// we had changes (added / removed) masters and therefore restart the workers. /// /// # Arguments /// * `node_type` - SparkNodeType (master/worker/history-server) /// * `hash` - NodeSelector hash /// * `version` - Current cluster version -/// * `master_node` - SparkNode master to retrieve master urls for worker start commands +/// * `master_node` - SparkNode master to retrieve master urls /// fn build_labels( node_type: &SparkNodeType, @@ -175,17 +178,28 @@ fn build_labels( labels.insert(HASH_LABEL.to_string(), hash.to_string()); labels.insert(VERSION_LABEL.to_string(), version.to_string()); + labels.insert( + SPARK_MASTER_URLS_HASH_LABEL.to_string(), + get_hashed_master_urls(master_node), + ); + + labels +} + +/// Get all master urls and hash them. This is required to keep track of which workers +/// are connected to which masters. In case masters are added / deleted, this hash changes +/// and we need to restart the worker pods to keep them up to date with all known masters. +/// +/// # Arguments +/// * `master_node` - SparkNode master to retrieve master urls for pod label hash +/// +pub fn get_hashed_master_urls(master_node: &SparkNode) -> String { let master_urls = config::get_master_urls(master_node); let mut hasher = DefaultHasher::new(); for url in master_urls { url.hash(&mut hasher); } - labels.insert( - SPARK_MASTER_URLS_LABEL.to_string(), - hasher.finish().to_string(), - ); - - labels + hasher.finish().to_string() } /// All pod names follow a simple pattern: --- From 8858fac14482ff6bcb237260555d35a3cb0ef700 Mon Sep 17 00:00:00 2001 From: maltesander Date: Mon, 8 Mar 2021 13:24:57 +0100 Subject: [PATCH 06/10] Only write master_url_hash into worker nodes --- operator/src/lib.rs | 11 ++++++++--- operator/src/pod_utils.rs | 12 +++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/operator/src/lib.rs b/operator/src/lib.rs index f126ab9..82c6b08 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -84,7 +84,7 @@ impl PodInformation { /// /// * `node_type` - Optional SparkNodeType (master/worker/history-server); /// - pub fn get_all_pods(&self, node_type: Option) -> Vec { + fn get_all_pods(&self, node_type: Option) -> Vec { let mut pods: Vec = vec![]; match node_type { @@ -306,9 +306,14 @@ impl SparkState { Ok(ReconcileFunctionAction::Continue) } - /// Read all cluster specific pods. Check for valid labels such as HASH (required to match a certain selector) or TYPE (which indicates master/worker/history-server). + /// Read all cluster specific pods. Check for required valid labels such as: + /// - HASH (required to match a certain selector) + /// - TYPE (which indicates master/worker/history-server) + /// - VERSION (cluster version the pod was created for) + /// /// Remove invalid pods which are lacking (or have outdated) required labels. - /// Sort incoming valid pods into corresponding maps (hash -> Vec) for later usage for each node type (master/worker/history-server). + /// Sort incoming valid pods into corresponding maps (hash -> Vec) + /// for later usage for each node type (master/worker/history-server). pub async fn read_existing_pod_information(&mut self) -> SparkReconcileResult { trace!( "Reading existing pod information for {}", diff --git a/operator/src/pod_utils.rs b/operator/src/pod_utils.rs index 0fcbfe7..0ff8821 100644 --- a/operator/src/pod_utils.rs +++ b/operator/src/pod_utils.rs @@ -19,7 +19,7 @@ pub const TYPE_LABEL: &str = "spark.stackable.tech/type"; /// Pod label which indicates the cluster version it was created for pub const VERSION_LABEL: &str = "spark.stackable.tech/currentVersion"; /// Pod label which indicates the known master urls for a worker pod -pub const SPARK_MASTER_URLS_HASH_LABEL: &str = "spark.stackable.tech/masterUrls"; +pub const MASTER_URLS_HASH_LABEL: &str = "spark.stackable.tech/masterUrls"; /// Name of the config volume to store configmap data const CONFIG_VOLUME: &str = "config-volume"; @@ -178,10 +178,12 @@ fn build_labels( labels.insert(HASH_LABEL.to_string(), hash.to_string()); labels.insert(VERSION_LABEL.to_string(), version.to_string()); - labels.insert( - SPARK_MASTER_URLS_HASH_LABEL.to_string(), - get_hashed_master_urls(master_node), - ); + if node_type == &SparkNodeType::Worker { + labels.insert( + MASTER_URLS_HASH_LABEL.to_string(), + get_hashed_master_urls(master_node), + ); + } labels } From 8ae9b3457fcffb0160bfee1f5faf49e8d9d06d0b Mon Sep 17 00:00:00 2001 From: maltesander Date: Mon, 8 Mar 2021 16:38:31 +0100 Subject: [PATCH 07/10] Workers are restarted in a rolling fashion if masters are added / deleted --- operator/src/config.rs | 39 ++++++++++++++++++++------------------- operator/src/lib.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/operator/src/config.rs b/operator/src/config.rs index 582acc7..73e57fc 100644 --- a/operator/src/config.rs +++ b/operator/src/config.rs @@ -4,8 +4,6 @@ use stackable_spark_crd::{ }; use std::collections::HashMap; -const SPARK_URL_START: &str = "spark://"; - // basic for startup const SPARK_NO_DAEMONIZE: &str = "SPARK_NO_DAEMONIZE"; const SPARK_CONF_DIR: &str = "SPARK_CONF_DIR"; @@ -29,7 +27,7 @@ const SPARK_HISTORY_STORE_PATH: &str = "spark.history.store.path"; const SPARK_HISTORY_UI_PORT: &str = "spark.history.ui.port"; /// The worker start command needs to be extended with all known master nodes and ports. -/// The required URLs are in format: 'spark://:: Op for url in master_urls { if !adapted_command.is_empty() { adapted_command.push(','); + } else { + adapted_command.push_str("spark://"); } adapted_command.push_str(url.as_str()); } @@ -68,38 +68,36 @@ pub fn get_master_urls(master: &SparkNode) -> Vec { // conf properties have higher priority than env variables if let Some(conf) = &selector.config { if let Some(port) = get_master_port(SPARK_MASTER_PORT_CONF, conf) { - master_urls.push(format!( - "{}{}:{}", - SPARK_URL_START, selector.node_name, port - )); + master_urls.push(create_master_url(&selector.node_name, &port.to_string())); continue; } } else if let Some(env) = &selector.env { if let Some(port) = get_master_port(SPARK_MASTER_PORT_ENV, env) { - master_urls.push(format!( - "{}{}:{}", - SPARK_URL_START, selector.node_name, port - )); + master_urls.push(create_master_url(&selector.node_name, &port.to_string())); continue; } } else if let Some(port) = selector.master_port { - master_urls.push(format!( - "{}{}:{}", - SPARK_URL_START, selector.node_name, port - )); + master_urls.push(create_master_url(&selector.node_name, &port.to_string())); continue; } // TODO: default to default value in product conf - master_urls.push(format!( - "{}{}:{}", - SPARK_URL_START, selector.node_name, "7077" - )); + master_urls.push(create_master_url(&selector.node_name, "7077")); } master_urls } +/// Create master url in format: : +/// +/// # Arguments +/// * `node_name` - Master node_name / host name +/// * `port` - Port on which the master is running +/// +fn create_master_url(node_name: &str, port: &str) -> String { + format!("{}:{}", node_name, port) +} + /// Search for a master port in config properties or env variables /// /// # Arguments @@ -139,6 +137,7 @@ pub fn create_required_startup_env() -> Vec { /// 2) from node /// 3) from selector /// 4) from config properties +/// /// # Arguments /// * `spec` - SparkCluster spec for common properties /// * `selector` - SparkClusterSelector containing desired config properties @@ -192,6 +191,7 @@ pub fn get_config_properties( /// 2) from node /// 3) from selector /// 4) from environment variables +/// /// # Arguments /// * `selector` - SparkClusterSelector containing desired env variables /// @@ -236,6 +236,7 @@ pub fn get_env_variables(selector: &SparkNodeSelector) -> HashMap SparkReconcileResult { + if let Some(pod_info) = &self.pod_information { + let worker_pods = pod_info.get_all_pods(Some(SparkNodeType::Worker)); + for pod in &worker_pods { + if let Some(labels) = &pod.metadata.labels { + if let Some(label_hashed_master_urls) = + labels.get(pod_utils::MASTER_URLS_HASH_LABEL) + { + let current_hashed_master_urls = + pod_utils::get_hashed_master_urls(&self.spec.master); + if label_hashed_master_urls != ¤t_hashed_master_urls { + info!( + "Pod [{}] has an outdated '{}' [{}] - required is [{}], deleting it", + Meta::name(pod), + pod_utils::MASTER_URLS_HASH_LABEL, + label_hashed_master_urls, + current_hashed_master_urls, + ); + self.context.client.delete(pod).await?; + return Ok(ReconcileFunctionAction::Requeue(Duration::from_secs(10))); + } + } + } + } + } + Ok(ReconcileFunctionAction::Continue) + } + /// Process the cluster status version for upgrades/downgrades. pub async fn process_version(&mut self) -> SparkReconcileResult { // If we reach here it means all pods must be running on target_version. @@ -710,6 +738,8 @@ impl ReconciliationState for SparkState { .await? .then(self.reconcile_cluster(&SparkNodeType::HistoryServer)) .await? + .then(self.check_worker_master_urls()) + .await? .then(self.process_version()) .await }) From 5c02dc3ec61cc1374ac484cfb5da9d889b5aeb1e Mon Sep 17 00:00:00 2001 From: maltesander Date: Mon, 8 Mar 2021 16:55:59 +0100 Subject: [PATCH 08/10] Added spark.port.maxRetries configuration property --- README.adoc | 7 ++++++- crd/sparkcluster.crd.yaml | 2 ++ crd/src/lib.rs | 1 + examples/sparkcluster.example.v3.0.1.yaml | 1 + operator/src/config.rs | 8 ++++++++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 2564e92..1d2fbe7 100644 --- a/README.adoc +++ b/README.adoc @@ -75,13 +75,18 @@ There are three levels of configuration: |logDir |string -|The log folder for spark applications +|The log folder for spark applications (must created by user) |spark.history.fs.logDirectory=logDir, spark.eventLog.enabled=true, spark.eventLog.dir=logDir; |secret |string |A secret shared between nodes and required to submit applications via spark-submit |spark.authenticate=true, spark.authenticate.secret=secret; + +|maxPortRetries +|integer +|Maximum number of retries when binding to a port before giving up. When a port is given a specific value (non 0), each subsequent retry will increment the port used in the previous attempt by 1 before retrying. This essentially allows it to try a range of ports from the start port specified to port + maxRetries. +|spark.port.maxRetries |=== === Node type options diff --git a/crd/sparkcluster.crd.yaml b/crd/sparkcluster.crd.yaml index 3965ba1..874f3fc 100644 --- a/crd/sparkcluster.crd.yaml +++ b/crd/sparkcluster.crd.yaml @@ -135,6 +135,8 @@ spec: type: string logDir: type: string + maxPortRetries: + type: integer status: nullable: true properties: diff --git a/crd/src/lib.rs b/crd/src/lib.rs index 0f3574c..402caa1 100644 --- a/crd/src/lib.rs +++ b/crd/src/lib.rs @@ -31,6 +31,7 @@ pub struct SparkClusterSpec { pub version: SparkVersion, pub secret: Option, pub log_dir: Option, + pub max_port_retries: Option, } impl SparkClusterSpec { diff --git a/examples/sparkcluster.example.v3.0.1.yaml b/examples/sparkcluster.example.v3.0.1.yaml index 5352ae8..fe5b189 100644 --- a/examples/sparkcluster.example.v3.0.1.yaml +++ b/examples/sparkcluster.example.v3.0.1.yaml @@ -20,3 +20,4 @@ spec: - nodeName: "mdesktop" instances: 1 version: "3.0.1" + maxPortRetries: 0 diff --git a/operator/src/config.rs b/operator/src/config.rs index 73e57fc..be0e741 100644 --- a/operator/src/config.rs +++ b/operator/src/config.rs @@ -12,6 +12,7 @@ const SPARK_EVENT_LOG_ENABLED: &str = "spark.eventLog.enabled"; const SPARK_EVENT_LOG_DIR: &str = "spark.eventLog.dir"; const SPARK_AUTHENTICATE: &str = "spark.authenticate"; const SPARK_AUTHENTICATE_SECRET: &str = "spark.authenticate.secret"; +const SPARK_PORT_MAX_RETRIES: &str = "spark.port.maxRetries"; // master const SPARK_MASTER_PORT_ENV: &str = "SPARK_MASTER_PORT"; const SPARK_MASTER_PORT_CONF: &str = "spark.master.port"; @@ -159,6 +160,13 @@ pub fn get_config_properties( config.insert(SPARK_AUTHENTICATE_SECRET.to_string(), secret.to_string()); } + if let Some(max_port_retries) = &spec.max_port_retries { + config.insert( + SPARK_PORT_MAX_RETRIES.to_string(), + max_port_retries.to_string(), + ); + } + // history server config.insert( SPARK_HISTORY_FS_LOG_DIRECTORY.to_string(), From 746ad9dc7ee1751f3a80ba0a99ab4d4e89a1b302 Mon Sep 17 00:00:00 2001 From: maltesander Date: Tue, 9 Mar 2021 10:37:31 +0100 Subject: [PATCH 09/10] removed clippy warning --- operator/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/operator/src/lib.rs b/operator/src/lib.rs index 5a5d094..1c6a9d5 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -19,7 +19,6 @@ use stackable_operator::conditions::ConditionStatus; use stackable_operator::config_map::create_config_map; use stackable_operator::controller::{Controller, ControllerStrategy, ReconciliationState}; use stackable_operator::error::OperatorResult; -use stackable_operator::krustlet::create_tolerations; use stackable_operator::reconcile::{ ReconcileFunctionAction, ReconcileResult, ReconciliationContext, }; From 803b0880b2e5f7187fb986a21652ad69e9037773 Mon Sep 17 00:00:00 2001 From: maltesander Date: Tue, 9 Mar 2021 11:55:22 +0100 Subject: [PATCH 10/10] Adapted to PR reviews --- operator/src/config.rs | 11 ++-- operator/src/lib.rs | 132 +++++++++++++++++++++-------------------- 2 files changed, 75 insertions(+), 68 deletions(-) diff --git a/operator/src/config.rs b/operator/src/config.rs index be0e741..6e3ebcf 100644 --- a/operator/src/config.rs +++ b/operator/src/config.rs @@ -28,8 +28,9 @@ const SPARK_HISTORY_STORE_PATH: &str = "spark.history.store.path"; const SPARK_HISTORY_UI_PORT: &str = "spark.history.ui.port"; /// The worker start command needs to be extended with all known master nodes and ports. -/// The required URLs are in format: ':::,: /// /// # Arguments /// * `node_type` - SparkNodeType (master/worker/history-server) @@ -44,10 +45,10 @@ pub fn adapt_worker_command(node_type: &SparkNodeType, master: &SparkNode) -> Op let master_urls = get_master_urls(master); for url in master_urls { - if !adapted_command.is_empty() { - adapted_command.push(','); - } else { + if adapted_command.is_empty() { adapted_command.push_str("spark://"); + } else { + adapted_command.push(','); } adapted_command.push_str(url.as_str()); } diff --git a/operator/src/lib.rs b/operator/src/lib.rs index 1c6a9d5..783dff6 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -517,69 +517,6 @@ impl SparkState { true } - pub async fn check_worker_master_urls(&self) -> SparkReconcileResult { - if let Some(pod_info) = &self.pod_information { - let worker_pods = pod_info.get_all_pods(Some(SparkNodeType::Worker)); - for pod in &worker_pods { - if let Some(labels) = &pod.metadata.labels { - if let Some(label_hashed_master_urls) = - labels.get(pod_utils::MASTER_URLS_HASH_LABEL) - { - let current_hashed_master_urls = - pod_utils::get_hashed_master_urls(&self.spec.master); - if label_hashed_master_urls != ¤t_hashed_master_urls { - info!( - "Pod [{}] has an outdated '{}' [{}] - required is [{}], deleting it", - Meta::name(pod), - pod_utils::MASTER_URLS_HASH_LABEL, - label_hashed_master_urls, - current_hashed_master_urls, - ); - self.context.client.delete(pod).await?; - return Ok(ReconcileFunctionAction::Requeue(Duration::from_secs(10))); - } - } - } - } - } - Ok(ReconcileFunctionAction::Continue) - } - - /// Process the cluster status version for upgrades/downgrades. - pub async fn process_version(&mut self) -> SparkReconcileResult { - // If we reach here it means all pods must be running on target_version. - // We can now set current_version to target_version (if target_version was set) and - // target_version to None - if let Some(status) = &self.status.clone() { - if let Some(target_version) = &status.target_version { - info!( - "Finished upgrade/downgrade to [{}]. Cluster ready!", - &target_version - ); - - self.status = self.set_target_version(None).await?.status; - self.status = self - .set_current_version(Some(&target_version)) - .await? - .status; - self.status = self - .set_upgrading_condition( - &status.conditions, - &format!( - "No change required [{:?}] is still the current_version", - target_version - ), - "", - ConditionStatus::False, - ) - .await? - .status; - } - } - - Ok(ReconcileFunctionAction::Continue) - } - /// Reconcile the cluster according to provided spec. Start with master nodes and continue to worker and history-server nodes. /// Create missing pods or delete excess pods to match the spec. /// @@ -663,6 +600,75 @@ impl SparkState { Ok(ReconcileFunctionAction::Continue) } + /// In spark stand alone, workers are started via script and require the master urls to connect to. + /// If masters change (added/deleted), workers need to be updated accordingly to be able + /// to fall back on other masters, if the primary master fails. + /// Therefore we always need to keep the workers updated in terms of available master urls. + /// Available master urls are hashed and stored as label in the worker pod. If the label differs + /// from the spec, we need to replace (delete and and) the workers in a rolling fashion. + pub async fn check_worker_master_urls(&self) -> SparkReconcileResult { + if let Some(pod_info) = &self.pod_information { + let worker_pods = pod_info.get_all_pods(Some(SparkNodeType::Worker)); + for pod in &worker_pods { + if let Some(labels) = &pod.metadata.labels { + if let Some(label_hashed_master_urls) = + labels.get(pod_utils::MASTER_URLS_HASH_LABEL) + { + let current_hashed_master_urls = + pod_utils::get_hashed_master_urls(&self.spec.master); + if label_hashed_master_urls != ¤t_hashed_master_urls { + info!( + "Pod [{}] has an outdated '{}' [{}] - required is [{}], deleting it", + Meta::name(pod), + pod_utils::MASTER_URLS_HASH_LABEL, + label_hashed_master_urls, + current_hashed_master_urls, + ); + self.context.client.delete(pod).await?; + return Ok(ReconcileFunctionAction::Requeue(Duration::from_secs(10))); + } + } + } + } + } + Ok(ReconcileFunctionAction::Continue) + } + + /// Process the cluster status version for upgrades/downgrades. + pub async fn process_version(&mut self) -> SparkReconcileResult { + // If we reach here it means all pods must be running on target_version. + // We can now set current_version to target_version (if target_version was set) and + // target_version to None + if let Some(status) = &self.status.clone() { + if let Some(target_version) = &status.target_version { + info!( + "Finished upgrade/downgrade to [{}]. Cluster ready!", + &target_version + ); + + self.status = self.set_target_version(None).await?.status; + self.status = self + .set_current_version(Some(&target_version)) + .await? + .status; + self.status = self + .set_upgrading_condition( + &status.conditions, + &format!( + "No change required [{:?}] is still the current_version", + target_version + ), + "", + ConditionStatus::False, + ) + .await? + .status; + } + } + + Ok(ReconcileFunctionAction::Continue) + } + /// Build a pod and create it /// /// # Arguments