diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b093fd5..488e2799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,21 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added +- Support for 1.15.0 ([#125]) +- Sensitive property key is setable via a secret ([#125]) + ### Changed +- Removed support for 1.13.2 ([#125]) +- Added/removed some default config settings that changed from 1.13 to 1.15 ([#125]) - `operator-rs` `0.3.0` → `0.4.0` ([#101]). - `stackable-zookeeper-crd`: `0.4.1` → `0.5.0` ([#101]). - Adapted pod image and container command to docker image ([#101]). - Adapted documentation to represent new workflow with docker images ([#101]). [#101]: https://github.com/stackabletech/nifi-operator/pull/101 +[#125]: https://github.com/stackabletech/nifi-operator/pull/125 ## [0.3.0] - 2021-10-27 diff --git a/deploy/crd/nificluster.crd.yaml b/deploy/crd/nificluster.crd.yaml index 3c43914c..ba0b1099 100644 --- a/deploy/crd/nificluster.crd.yaml +++ b/deploy/crd/nificluster.crd.yaml @@ -52,6 +52,10 @@ spec: minimum: 0.0 nullable: true type: integer + sensitivePropertyKeySecret: + type: string + required: + - sensitivePropertyKeySecret type: object configOverrides: additionalProperties: @@ -91,6 +95,10 @@ spec: minimum: 0.0 nullable: true type: integer + sensitivePropertyKeySecret: + type: string + required: + - sensitivePropertyKeySecret type: object configOverrides: additionalProperties: @@ -147,7 +155,7 @@ spec: type: object version: enum: - - 1.13.2 + - 1.15.0 type: string zookeeperReference: description: Contains all necessary information identify a Stackable managed ZooKeeper ensemble and build a connection string for it. The main purpose for this struct is for other operators that need to reference a ZooKeeper ensemble to use in their CRDs. This has the benefit of keeping references to Zookeeper ensembles consistent throughout the entire stack. @@ -224,12 +232,12 @@ spec: properties: current: enum: - - 1.13.2 + - 1.15.0 nullable: true type: string target: enum: - - 1.13.2 + - 1.15.0 nullable: true type: string type: object diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index c42f7dff..8d569b72 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -4,7 +4,7 @@ After installation, the CRD for this operator must be created: kubectl apply -f /etc/stackable/nifi-operator/crd/nificluster.crd.yaml -To create a single node Apache NiFi (v1.13.2) cluster with Prometheus metrics exposed on port 8428 and the WebUi on port 10000. As of now only one instance per node is supported: +To create a single node Apache NiFi (v1.15.0) cluster with Prometheus metrics exposed on port 8428 and the WebUi on port 10000. As of now only one instance per node is supported: cat < for NifiCluster { strum_macros::EnumString, )] pub enum NifiVersion { - #[serde(rename = "1.13.2")] - #[strum(serialize = "1.13.2")] - v1_13_2, - - // TODO: NiFi 1.14 does not work with this operator yet - // Therefore we skip using it in the CRD for now. Should be uncommented as soon as #82 is fixed. - //#[serde(rename = "1.14.0")] - #[serde(skip)] - #[strum(serialize = "1.14.0")] - v1_14_0, + #[serde(rename = "1.15.0")] + #[strum(serialize = "1.15.0")] + v1_15_0, } impl Versioning for NifiVersion { @@ -143,6 +138,7 @@ pub struct NifiConfig { pub http_port: Option, pub protocol_port: Option, pub load_balance_port: Option, + pub sensitive_property_key_secret: String, } impl Configuration for NifiConfig { @@ -160,6 +156,10 @@ impl Configuration for NifiConfig { Some(metrics_port.to_string()), ); } + result.insert( + NIFI_SENSITIVE_PROPS_KEY.to_string(), + Some(self.sensitive_property_key_secret.to_string()), + ); Ok(result) } @@ -214,23 +214,14 @@ mod tests { #[test] fn test_zookeeper_version_versioning() { assert_eq!( - NifiVersion::v1_13_2.versioning_state(&NifiVersion::v1_14_0), - VersioningState::ValidUpgrade - ); - assert_eq!( - NifiVersion::v1_14_0.versioning_state(&NifiVersion::v1_13_2), - VersioningState::ValidDowngrade - ); - assert_eq!( - NifiVersion::v1_13_2.versioning_state(&NifiVersion::v1_13_2), + NifiVersion::v1_15_0.versioning_state(&NifiVersion::v1_15_0), VersioningState::NoOp ); } #[test] fn test_version_conversion() { - NifiVersion::from_str("1.13.2").unwrap(); - NifiVersion::from_str("1.14.0").unwrap(); + NifiVersion::from_str("1.15.0").unwrap(); NifiVersion::from_str("1.2.3").unwrap_err(); } } diff --git a/rust/operator/src/config.rs b/rust/operator/src/config.rs index 8354a38f..9bc78274 100644 --- a/rust/operator/src/config.rs +++ b/rust/operator/src/config.rs @@ -25,6 +25,8 @@ pub fn build_bootstrap_conf() -> String { bootstrap.insert("java", "java".to_string()); // Username to use when running NiFi. This value will be ignored on Windows. bootstrap.insert("run.as", "".to_string()); + // Preserve shell environment while runnning as "run.as" user + bootstrap.insert("preserve.environment", "false".to_string()); // Configure where NiFi's lib and conf directories live bootstrap.insert("lib.dir", "./lib".to_string()); bootstrap.insert("conf.dir", "./conf".to_string()); @@ -213,6 +215,22 @@ pub fn build_nifi_properties( ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE".to_string(), ); + // Repository Encryption properties override individual repository implementation properties + properties.insert( + "nifi.repository.encryption.protocol.version", + "".to_string(), + ); + properties.insert("nifi.repository.encryption.key.id", "".to_string()); + properties.insert("nifi.repository.encryption.key.provider", "".to_string()); + properties.insert( + "nifi.repository.encryption.key.provider.keystore.location", + "".to_string(), + ); + properties.insert( + "nifi.repository.encryption.key.provider.keystore.password", + "".to_string(), + ); + // FlowFile Repository properties.insert( "nifi.flowfile.repository.implementation", @@ -231,16 +249,6 @@ pub fn build_nifi_properties( "20 secs".to_string(), ); properties.insert("nifi.flowfile.repository.always.sync", "false".to_string()); - properties.insert( - "nifi.flowfile.repository.encryption.key.provider.implementation", - "".to_string(), - ); - properties.insert( - "nifi.flowfile.repository.encryption.key.provider.location", - "".to_string(), - ); - properties.insert("nifi.flowfile.repository.encryption.key.id", "".to_string()); - properties.insert("nifi.flowfile.repository.encryption.key", "".to_string()); properties.insert( "nifi.flowfile.repository.retain.orphaned.flowfiles", "true".to_string(), @@ -279,35 +287,12 @@ pub fn build_nifi_properties( "nifi.content.viewer.url", "../nifi-content-viewer/".to_string(), ); - properties.insert( - "nifi.content.repository.encryption.key.provider.implementation", - "".to_string(), - ); - properties.insert( - "nifi.content.repository.encryption.key.provider.location", - "".to_string(), - ); - properties.insert("nifi.content.repository.encryption.key.id", "".to_string()); - properties.insert("nifi.content.repository.encryption.key", "".to_string()); // Provenance Repository Properties properties.insert( "nifi.provenance.repository.implementation", "org.apache.nifi.provenance.WriteAheadProvenanceRepository".to_string(), ); - properties.insert( - "nifi.provenance.repository.encryption.key.provider.implementation", - "".to_string(), - ); - properties.insert( - "nifi.provenance.repository.encryption.key.provider.location", - "".to_string(), - ); - properties.insert( - "nifi.provenance.repository.encryption.key.id", - "".to_string(), - ); - properties.insert("nifi.provenance.repository.encryption.key", "".to_string()); // Persistent Provenance Repository Properties properties.insert( @@ -388,6 +373,20 @@ pub fn build_nifi_properties( "1 min".to_string(), ); + // QuestDB Status History Repository Properties + properties.insert( + "nifi.status.repository.questdb.persist.node.days", + "14".to_string(), + ); + properties.insert( + "nifi.status.repository.questdb.persist.component.days", + "3".to_string(), + ); + properties.insert( + "nifi.status.repository.questdb.persist.location", + "./status_repository".to_string(), + ); + // Site to Site properties properties.insert("nifi.remote.input.host", node_name.to_string()); properties.insert("nifi.remote.input.secure", "false".to_string()); @@ -431,18 +430,29 @@ pub fn build_nifi_properties( properties.insert("nifi.web.proxy.host", "".to_string()); properties.insert("nifi.web.max.content.size", "".to_string()); properties.insert("nifi.web.max.requests.per.second", "30000".to_string()); + properties.insert( + "nifi.web.max.access.token.requests.per.second", + "25".to_string(), + ); + properties.insert("nifi.web.request.timeout", "60 secs".to_string()); + properties.insert("nifi.web.request.ip.whitelist", "".to_string()); properties.insert("nifi.web.should.send.server.version", "true".to_string()); + // Include or Exclude TLS Cipher Suites for HTTPS + properties.insert("nifi.web.https.ciphersuites.include", "".to_string()); + properties.insert("nifi.web.https.ciphersuites.exclude", "".to_string()); + // security properties - properties.insert("nifi.sensitive.props.key", "".to_string()); + properties.insert("nifi.sensitive.props.key", "".to_string()); // this property is later set from a secret properties.insert("nifi.sensitive.props.key.protected", "".to_string()); properties.insert( "nifi.sensitive.props.algorithm", - "PBEWITHMD5AND256BITAES-CBC-OPENSSL".to_string(), + "NIFI_PBKDF2_AES_GCM_256".to_string(), ); - properties.insert("nifi.sensitive.props.provider", "BC".to_string()); properties.insert("nifi.sensitive.props.additional.keys", "".to_string()); + properties.insert("nifi.security.autoreload.enabled", "false".to_string()); + properties.insert("nifi.security.autoreload.interval", "10 secs".to_string()); properties.insert("nifi.security.keystore", "".to_string()); properties.insert("nifi.security.keystoreType", "".to_string()); properties.insert("nifi.security.keystorePasswd", "".to_string()); @@ -459,6 +469,10 @@ pub fn build_nifi_properties( "false".to_string(), ); properties.insert("nifi.security.user.login.identity.provider", "".to_string()); + properties.insert( + "nifi.security.user.jws.key.rotation.period", + "PT1H".to_string(), + ); properties.insert("nifi.security.ocsp.responder.url", "".to_string()); properties.insert("nifi.security.ocsp.responder.certificate", "".to_string()); @@ -646,6 +660,7 @@ pub fn build_nifi_properties( properties.insert("nifi.zookeeper.security.truststore", "".to_string()); properties.insert("nifi.zookeeper.security.truststoreType", "".to_string()); properties.insert("nifi.zookeeper.security.truststorePasswd", "".to_string()); + properties.insert("nifi.zookeeper.jute.maxbuffer", "".to_string()); // Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management // Values supported for nifi.zookeeper.auth.type are "default", which will apply world/anyone rights on znodes @@ -699,6 +714,36 @@ pub fn build_nifi_properties( ".90".to_string(), ); + // runtime monitoring properties + properties.insert("nifi.monitor.long.running.task.schedule", "".to_string()); + properties.insert("nifi.monitor.long.running.task.threshold", "".to_string()); + + // Create automatic diagnostics when stopping/restarting NiFi. + + // Enable automatic diagnostic at shutdown. + properties.insert("nifi.diagnostics.on.shutdown.enabled", "false".to_string()); + + // Include verbose diagnostic information. + properties.insert("nifi.diagnostics.on.shutdown.verbose", "false".to_string()); + + // The location of the diagnostics folder. + properties.insert( + "nifi.diagnostics.on.shutdown.directory", + "./diagnostics".to_string(), + ); + + // The maximum number of files permitted in the directory. If the limit is exceeded, the oldest files are deleted. + properties.insert( + "nifi.diagnostics.on.shutdown.max.filecount", + "10".to_string(), + ); + + // The diagnostics folder's maximum permitted size in bytes. If the limit is exceeded, the oldest files are deleted. + properties.insert( + "nifi.diagnostics.on.shutdown.max.directory.size", + "10 MB".to_string(), + ); + format_properties(properties) } diff --git a/rust/operator/src/lib.rs b/rust/operator/src/lib.rs index 8c7c9b9c..4febe813 100644 --- a/rust/operator/src/lib.rs +++ b/rust/operator/src/lib.rs @@ -14,7 +14,8 @@ use async_trait::async_trait; use futures::Future; use stackable_nifi_crd::{ NifiCluster, NifiRole, NifiSpec, APP_NAME, MANAGED_BY, NIFI_CLUSTER_LOAD_BALANCE_PORT, - NIFI_CLUSTER_METRICS_PORT, NIFI_CLUSTER_NODE_PROTOCOL_PORT, NIFI_WEB_HTTP_PORT, + NIFI_CLUSTER_METRICS_PORT, NIFI_CLUSTER_NODE_PROTOCOL_PORT, NIFI_SENSITIVE_PROPS_KEY, + NIFI_WEB_HTTP_PORT, }; use stackable_operator::builder::{ContainerBuilder, ObjectMetaBuilder, PodBuilder, VolumeBuilder}; use stackable_operator::client::Client; @@ -23,7 +24,9 @@ use stackable_operator::error::OperatorResult; use stackable_operator::identity::{ LabeledPodIdentityFactory, NodeIdentity, PodIdentity, PodToNodeMapping, }; -use stackable_operator::k8s_openapi::api::core::v1::{ConfigMap, EnvVar, Pod}; +use stackable_operator::k8s_openapi::api::core::v1::{ + ConfigMap, EnvVar, EnvVarSource, Pod, SecretKeySelector, +}; use stackable_operator::kube::api::ListParams; use stackable_operator::kube::Api; use stackable_operator::kube::ResourceExt; @@ -442,6 +445,18 @@ impl NifiState { } } + let secret = validated_config + .get(&PropertyNameKind::Env) + .and_then(|m| m.get(NIFI_SENSITIVE_PROPS_KEY)); + + if let Some(s) = secret { + env_vars.push(env_var_from_secret( + NIFI_SENSITIVE_PROPS_KEY, + s, + "nifiSensitivePropsKey", + )); + } + let pod_name = name_utils::build_resource_name( pod_id.app(), pod_id.instance(), @@ -469,7 +484,7 @@ impl NifiState { // we use the copy_assets.sh script here to copy everything from the "STACKABLE_TMP_CONFIG" // folder to the "conf" folder in the nifi package. container_builder.args(vec![format!( - "/stackable/bin/copy_assets {}; {} {}", + "/stackable/bin/copy_assets {}; /stackable/bin/update_config; {} {}", STACKABLE_TMP_CONFIG, "bin/nifi.sh", "run" )]); container_builder.add_env_vars(env_vars); @@ -702,6 +717,21 @@ impl NifiState { } } +fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> EnvVar { + EnvVar { + name: String::from(var_name), + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: Some(String::from(secret)), + key: String::from(secret_key), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } +} + impl ReconciliationState for NifiState { type Error = error::NifiError;