From 7339bee1e3f1867c94994d928204d22ef1c2e773 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 20 Oct 2025 11:35:05 +0200 Subject: [PATCH 1/5] refactor: Split into Deployment and DaemonSet The splits the deployment of the secret-operator as a whole into two parts: - The controller is deployed via a Deployment which ensures only a single instance of the secret-operator in controller mode is running in a Kubernetes cluster. This can potentially lead to perfomance issues and as such should be monitored going forward. - The CSI server is deployed via a DaemonSet (unchanged) as this server is needed on every node to provision requested secret volumes. This refactor is introduced in preparation for #634, in which only a single instance of the CRD conversion webhook must exist as otherwise TLS certificate verification will fail with multiple available certificates. --- Tiltfile | 3 +- .../templates/controller-deployment.yaml | 96 ++++++++++ ...emonset.yaml => csi-server-daemonset.yaml} | 18 +- rust/operator-binary/src/main.rs | 169 +++++++++++------- .../src/truststore_controller.rs | 20 +-- 5 files changed, 223 insertions(+), 83 deletions(-) create mode 100644 deploy/helm/secret-operator/templates/controller-deployment.yaml rename deploy/helm/secret-operator/templates/{daemonset.yaml => csi-server-daemonset.yaml} (87%) diff --git a/Tiltfile b/Tiltfile index fb84bcc0..033900c7 100644 --- a/Tiltfile +++ b/Tiltfile @@ -26,6 +26,7 @@ if os.path.exists('result'): # oci.stackable.tech/sandbox/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of # oci.stackable.tech/sandbox/opa-operator:0.0.0-dev (which does not exist) k8s_kind('Deployment', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') +k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') # Exclude stale CRDs from Helm chart, and apply the rest helm_crds, helm_non_crds = filter_yaml( @@ -34,7 +35,7 @@ helm_crds, helm_non_crds = filter_yaml( name=operator_name, namespace="stackable-operators", set=[ - 'image.repository=' + registry + '/' + operator_name, + 'secretOperator.image.repository=' + registry + '/' + operator_name, ], ), api_version = "^apiextensions\\.k8s\\.io/.*$", diff --git a/deploy/helm/secret-operator/templates/controller-deployment.yaml b/deploy/helm/secret-operator/templates/controller-deployment.yaml new file mode 100644 index 00000000..79222260 --- /dev/null +++ b/deploy/helm/secret-operator/templates/controller-deployment.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "operator.fullname" . }}-controller + labels: + {{- include "operator.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "operator.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + internal.stackable.tech/image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "operator.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + # NOTE (@Techassi): Does it maybe make sense to have two different service accounts? + serviceAccountName: {{ include "operator.fullname" . }}-serviceaccount + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ include "operator.appname" . }}-controller + securityContext: + {{- toYaml .Values.secretOperator.securityContext | nindent 12 }} + image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.secretOperator.image.pullPolicy }} + resources: + {{ .Values.secretOperator.resources | toYaml | nindent 12 }} + # The arguments passed to the command being run in the container. The final command will + # look like `secret-operator run controller [OPTIONS]`. The controller needs to only run + # once in a Kubernetes cluster and as such is deployed as a Deployment. + args: + - run + - controller + env: + # The following env vars are passed as clap (think CLI) arguments to the operator. + # They are picked up by clap using the structs defied in the operator. + # (which is turn pulls in https://github.com/stackabletech/operator-rs/blob/main/crates/stackable-operator/src/cli.rs) + # You can read there about the expected values and purposes. + + # Sometimes products need to know the operator image, e.g. the opa-bundle-builder OPA + # sidecar uses the operator image. + - name: OPERATOR_IMAGE + # Tilt can use annotations as image paths, but not env variables + valueFrom: + fieldRef: + fieldPath: metadata.annotations['internal.stackable.tech/image'] + + # Namespace the operator Pod is running in, e.g. used to construct the conversion + # webhook endpoint. + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + # The name of the Kubernetes Service that point to the operator Pod, e.g. used to + # construct the conversion webhook endpoint. + - name: OPERATOR_SERVICE_NAME + value: {{ include "operator.fullname" . }} + + # Operators need to know the node name they are running on, to e.g. discover the + # Kubernetes domain name from the kubelet API. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + + {{- if .Values.kubernetesClusterDomain }} + - name: KUBERNETES_CLUSTER_DOMAIN + value: {{ .Values.kubernetesClusterDomain | quote }} + {{- end }} + {{- include "telemetry.envVars" . | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} diff --git a/deploy/helm/secret-operator/templates/daemonset.yaml b/deploy/helm/secret-operator/templates/csi-server-daemonset.yaml similarity index 87% rename from deploy/helm/secret-operator/templates/daemonset.yaml rename to deploy/helm/secret-operator/templates/csi-server-daemonset.yaml index 9da10540..23ae8158 100644 --- a/deploy/helm/secret-operator/templates/daemonset.yaml +++ b/deploy/helm/secret-operator/templates/csi-server-daemonset.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "operator.fullname" . }}-daemonset + name: {{ include "operator.fullname" . }}-csi-server labels: {{- include "operator.labels" . | nindent 4 }} spec: @@ -11,10 +11,11 @@ spec: {{- include "operator.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} - {{- end }} + internal.stackable.tech/image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} labels: {{- include "operator.selectorLabels" . | nindent 8 }} spec: @@ -22,17 +23,24 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + # NOTE (@Techassi): Does it maybe make sense to have two different service accounts? serviceAccountName: {{ include "operator.fullname" . }}-serviceaccount securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - - name: {{ include "operator.appname" . }} + - name: {{ include "operator.appname" . }}-csi-server securityContext: {{- toYaml .Values.secretOperator.securityContext | nindent 12 }} image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.secretOperator.image.pullPolicy }} resources: {{ .Values.secretOperator.resources | toYaml | nindent 12 }} + # The arguments passed to the command being run in the container. The final command will + # look like `secret-operator run csi-server [OPTIONS]`. The CSI server needs to run on + # every Kubernetes cluster node and as such is deployed as a DaemonSet. + args: + - run + - csi-server env: # The following env vars are passed as clap (think CLI) arguments to the operator. # They are picked up by clap using the structs defied in the operator. diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 5c9b3b81..5e061805 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -2,7 +2,7 @@ // This will need changes in our and upstream error types. #![allow(clippy::result_large_err)] -use std::{os::unix::prelude::FileTypeExt, path::PathBuf, pin::pin}; +use std::{os::unix::prelude::FileTypeExt, path::PathBuf}; use anyhow::Context; use clap::Parser; @@ -16,7 +16,9 @@ use grpc::csi::v1::{ }; use stackable_operator::{ YamlSchema, - cli::{CommonOptions, ProductOperatorRun}, + cli::{Command, CommonOptions, ProductOperatorRun}, + client::Client, + namespace::WatchNamespace, shared::yaml::SerializeOptions, telemetry::Tracing, }; @@ -40,14 +42,33 @@ pub const OPERATOR_NAME: &str = "secrets.stackable.tech"; #[derive(clap::Parser)] #[clap(author, version)] -struct Opts { +struct Cli { #[clap(subcommand)] - cmd: stackable_operator::cli::Command, + cmd: Command, } #[derive(clap::Parser)] struct SecretOperatorRun { - #[clap(long, env)] + /// The run mode in which this operator should run in. + #[command(subcommand)] + mode: RunMode, + + #[clap(flatten)] + common: ProductOperatorRun, +} + +#[derive(Debug, clap::Subcommand)] +enum RunMode { + /// Run the CSI server, one per Kubernetes cluster node. + CsiServer(CsiServerArguments), + + /// Run the controller, one per Kubernetes cluster. + Controller, +} + +#[derive(Debug, clap::Args)] +struct CsiServerArguments { + #[arg(long, env)] csi_endpoint: PathBuf, /// Unprivileged mode disables any features that require running secret-operator in a privileged container. @@ -56,11 +77,8 @@ struct SecretOperatorRun { /// - Secret volumes will be stored on disk, rather than in a ramdisk /// /// Unprivileged mode is EXPERIMENTAL and heavily discouraged, since it increases the risk of leaking secrets. - #[clap(long, env)] + #[arg(long, env)] privileged: bool, - - #[clap(flatten)] - common: ProductOperatorRun, } mod built_info { @@ -69,29 +87,28 @@ mod built_info { #[tokio::main] async fn main() -> anyhow::Result<()> { - let opts = Opts::parse(); - match opts.cmd { - stackable_operator::cli::Command::Crd => { + let cli = Cli::parse(); + + match cli.cmd { + Command::Crd => { SecretClass::merged_crd(crd::SecretClassVersion::V1Alpha1)? .print_yaml_schema(built_info::PKG_VERSION, SerializeOptions::default())?; TrustStore::merged_crd(crd::TrustStoreVersion::V1Alpha1)? .print_yaml_schema(built_info::PKG_VERSION, SerializeOptions::default())?; } - stackable_operator::cli::Command::Run(SecretOperatorRun { - csi_endpoint, - privileged, - common: - ProductOperatorRun { - common: - CommonOptions { - telemetry, - cluster_info, - }, - product_config: _, - watch_namespace, - operator_environment: _, - }, - }) => { + Command::Run(SecretOperatorRun { common, mode }) => { + let ProductOperatorRun { + operator_environment: _, + product_config: _, + watch_namespace, + common, + } = common; + + let CommonOptions { + telemetry, + cluster_info, + } = common; + // NOTE (@NickLarsenNZ): Before stackable-telemetry was used: // - The console log level was set by `SECRET_PROVISIONER_LOG`, and is now `CONSOLE_LOG` (when using Tracing::pre_configured). // - The file log level was set by `SECRET_PROVISIONER_LOG`, and is now set via `FILE_LOG` (when using Tracing::pre_configured). @@ -113,46 +130,70 @@ async fn main() -> anyhow::Result<()> { &cluster_info, ) .await?; - if csi_endpoint - .symlink_metadata() - .is_ok_and(|meta| meta.file_type().is_socket()) - { - let _ = std::fs::remove_file(&csi_endpoint); - } - let mut sigterm = signal(SignalKind::terminate())?; - let csi_server = pin!( - Server::builder() - .add_service( - tonic_reflection::server::Builder::configure() - .include_reflection_service(true) - .register_encoded_file_descriptor_set(grpc::FILE_DESCRIPTOR_SET_BYTES) - .build_v1()?, - ) - .add_service(IdentityServer::new(SecretProvisionerIdentity)) - .add_service(ControllerServer::new(SecretProvisionerController { - client: client.clone(), - })) - .add_service(NodeServer::new(SecretProvisionerNode { - client: client.clone(), - node_name: cluster_info.kubernetes_node_name.to_owned(), + + match mode { + RunMode::CsiServer(CsiServerArguments { + csi_endpoint, + privileged, + }) => { + run_csi_server( + csi_endpoint, + cluster_info.kubernetes_node_name, privileged, - })) - .serve_with_incoming_shutdown( - UnixListenerStream::new( - uds_bind_private(csi_endpoint) - .context("failed to bind CSI listener")?, - ) - .map_ok(TonicUnixStream), - sigterm.recv().map(|_| ()), + client, ) - ); - let truststore_controller = - pin!(truststore_controller::start(&client, &watch_namespace).map(Ok)); - futures::future::select(csi_server, truststore_controller) - .await - .factor_first() - .0?; + .await? + } + RunMode::Controller => run_controller(watch_namespace, client).await, + } } } + Ok(()) } + +async fn run_csi_server( + csi_endpoint: PathBuf, + node_name: String, + privileged: bool, + client: Client, +) -> anyhow::Result<()> { + if csi_endpoint + .symlink_metadata() + .is_ok_and(|meta| meta.file_type().is_socket()) + { + let _ = std::fs::remove_file(&csi_endpoint); + } + + let mut sigterm = signal(SignalKind::terminate())?; + + let csi_server = Server::builder() + .add_service( + tonic_reflection::server::Builder::configure() + .include_reflection_service(true) + .register_encoded_file_descriptor_set(grpc::FILE_DESCRIPTOR_SET_BYTES) + .build_v1()?, + ) + .add_service(IdentityServer::new(SecretProvisionerIdentity)) + .add_service(ControllerServer::new(SecretProvisionerController { + client: client.clone(), + })) + .add_service(NodeServer::new(SecretProvisionerNode { + privileged, + node_name, + client, + })) + .serve_with_incoming_shutdown( + UnixListenerStream::new( + uds_bind_private(csi_endpoint).context("failed to bind CSI listener")?, + ) + .map_ok(TonicUnixStream), + sigterm.recv().map(|_| ()), + ); + + csi_server.await.context("failed to run the CSI server") +} + +async fn run_controller(watch_namespace: WatchNamespace, client: Client) { + truststore_controller::start(client, &watch_namespace).await +} diff --git a/rust/operator-binary/src/truststore_controller.rs b/rust/operator-binary/src/truststore_controller.rs index 28bec96d..3c9f599c 100644 --- a/rust/operator-binary/src/truststore_controller.rs +++ b/rust/operator-binary/src/truststore_controller.rs @@ -43,10 +43,10 @@ use crate::{ const CONTROLLER_NAME: &str = "truststore"; const FULL_CONTROLLER_NAME: &str = concatcp!(CONTROLLER_NAME, ".", OPERATOR_NAME); -pub async fn start(client: &stackable_operator::client::Client, watch_namespace: &WatchNamespace) { +pub async fn start(client: stackable_operator::client::Client, watch_namespace: &WatchNamespace) { let (secretclasses, secretclasses_writer) = reflector::store(); let controller = Controller::new( - watch_namespace.get_api::>(client), + watch_namespace.get_api::>(&client), watcher::Config::default(), ); let truststores = controller.store(); @@ -82,16 +82,16 @@ pub async fn start(client: &stackable_operator::client::Client, watch_namespace: ) // TODO: merge this into the other ConfigMap watch .owns( - watch_namespace.get_api::>(client), + watch_namespace.get_api::>(&client), watcher::Config::default(), ) // TODO: merge this into the other Secret watch .owns( - watch_namespace.get_api::>(client), + watch_namespace.get_api::>(&client), watcher::Config::default(), ) .watches( - watch_namespace.get_api::>(client), + watch_namespace.get_api::>(&client), watcher::Config::default(), secretclass_dependency_watch_mapper( truststores.clone(), @@ -100,7 +100,7 @@ pub async fn start(client: &stackable_operator::client::Client, watch_namespace: ), ) .watches( - watch_namespace.get_api::>(client), + watch_namespace.get_api::>(&client), watcher::Config::default(), secretclass_dependency_watch_mapper( truststores, @@ -108,13 +108,7 @@ pub async fn start(client: &stackable_operator::client::Client, watch_namespace: |secretclass, secret| secretclass.spec.backend.refers_to_secret(secret), ), ) - .run( - reconcile, - error_policy, - Arc::new(Ctx { - client: client.clone(), - }), - ) + .run(reconcile, error_policy, Arc::new(Ctx { client })) .for_each_concurrent(16, move |res| { let event_recorder = event_recorder.clone(); async move { From 0d3f6addca810c40d797745612b2f7d4a841b80f Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 20 Oct 2025 13:22:20 +0200 Subject: [PATCH 2/5] chore: Add changelog entry --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb39576d..aa5531d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ All notable changes to this project will be documented in this file. ### Changed +- Split operator deployment into Deployment and DaemonSet ([#645]). + - Introduce two different modes: `csi-server` and `controller`. + - The CSI server is deployed via a DaemonSet to be available on every node. + - The controller is deployed via a Deployment with a single replica. - Version CRD structs and enums as v1alpha1 ([#636]). - BREAKING: Rearrange values to be somewhat consistent with the listener-operator value changes ([#641]). - `image.repository` has been moved to `secretOperator.image.repository`. @@ -28,6 +32,7 @@ All notable changes to this project will be documented in this file. [#641]: https://github.com/stackabletech/secret-operator/pull/641 [#642]: https://github.com/stackabletech/secret-operator/pull/642 [#643]: https://github.com/stackabletech/secret-operator/pull/643 +[#645]: https://github.com/stackabletech/secret-operator/pull/645 ## [25.7.0] - 2025-07-23 From 0d64dd161272da024d3124a45aaae1ee72def439 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 20 Oct 2025 13:52:40 +0200 Subject: [PATCH 3/5] chore: Update comment --- .../helm/secret-operator/templates/controller-deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/helm/secret-operator/templates/controller-deployment.yaml b/deploy/helm/secret-operator/templates/controller-deployment.yaml index 79222260..43a7154b 100644 --- a/deploy/helm/secret-operator/templates/controller-deployment.yaml +++ b/deploy/helm/secret-operator/templates/controller-deployment.yaml @@ -37,7 +37,8 @@ spec: {{ .Values.secretOperator.resources | toYaml | nindent 12 }} # The arguments passed to the command being run in the container. The final command will # look like `secret-operator run controller [OPTIONS]`. The controller needs to only run - # once in a Kubernetes cluster and as such is deployed as a Deployment. + # once in a Kubernetes cluster and as such is deployed as a Deployment with a single + # replica. args: - run - controller From 767727309a037aa429b14fe8e21265f82969f3ab Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 21 Oct 2025 12:11:21 +0200 Subject: [PATCH 4/5] refactor: Adjust values.yaml file to be closer to listener-operator --- Tiltfile | 2 +- .../templates/controller-deployment.yaml | 28 ++-- ...et.yaml => csi-node-driver-daemonset.yaml} | 56 ++++---- deploy/helm/secret-operator/values.yaml | 129 ++++++++++-------- rust/operator-binary/src/main.rs | 6 +- 5 files changed, 122 insertions(+), 99 deletions(-) rename deploy/helm/secret-operator/templates/{csi-server-daemonset.yaml => csi-node-driver-daemonset.yaml} (67%) diff --git a/Tiltfile b/Tiltfile index 033900c7..d2295bde 100644 --- a/Tiltfile +++ b/Tiltfile @@ -35,7 +35,7 @@ helm_crds, helm_non_crds = filter_yaml( name=operator_name, namespace="stackable-operators", set=[ - 'secretOperator.image.repository=' + registry + '/' + operator_name, + 'image.repository=' + registry + '/' + operator_name, ], ), api_version = "^apiextensions\\.k8s\\.io/.*$", diff --git a/deploy/helm/secret-operator/templates/controller-deployment.yaml b/deploy/helm/secret-operator/templates/controller-deployment.yaml index 43a7154b..dca71496 100644 --- a/deploy/helm/secret-operator/templates/controller-deployment.yaml +++ b/deploy/helm/secret-operator/templates/controller-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "operator.fullname" . }}-controller + name: {{ include "operator.fullname" . }} labels: {{- include "operator.labels" . | nindent 4 }} spec: @@ -12,29 +12,29 @@ spec: template: metadata: annotations: - internal.stackable.tech/image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" - {{- with .Values.podAnnotations }} + internal.stackable.tech/image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- with .Values.controllerService.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: {{- include "operator.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with .Values.image.pullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} # NOTE (@Techassi): Does it maybe make sense to have two different service accounts? serviceAccountName: {{ include "operator.fullname" . }}-serviceaccount securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml .Values.controllerService.podSecurityContext | nindent 8 }} containers: - - name: {{ include "operator.appname" . }}-controller + - name: {{ include "operator.appname" . }} securityContext: - {{- toYaml .Values.secretOperator.securityContext | nindent 12 }} - image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.secretOperator.image.pullPolicy }} + {{- toYaml .Values.controllerService.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} resources: - {{ .Values.secretOperator.resources | toYaml | nindent 12 }} + {{ .Values.controllerService.resources | toYaml | nindent 12 }} # The arguments passed to the command being run in the container. The final command will # look like `secret-operator run controller [OPTIONS]`. The controller needs to only run # once in a Kubernetes cluster and as such is deployed as a Deployment with a single @@ -80,18 +80,18 @@ spec: value: {{ .Values.kubernetesClusterDomain | quote }} {{- end }} {{- include "telemetry.envVars" . | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- with .Values.controllerService.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with .Values.controllerService.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with .Values.controllerService.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.priorityClassName }} + {{- with .Values.controllerService.priorityClassName }} priorityClassName: {{ . }} {{- end }} diff --git a/deploy/helm/secret-operator/templates/csi-server-daemonset.yaml b/deploy/helm/secret-operator/templates/csi-node-driver-daemonset.yaml similarity index 67% rename from deploy/helm/secret-operator/templates/csi-server-daemonset.yaml rename to deploy/helm/secret-operator/templates/csi-node-driver-daemonset.yaml index 23ae8158..bfbd42f2 100644 --- a/deploy/helm/secret-operator/templates/csi-server-daemonset.yaml +++ b/deploy/helm/secret-operator/templates/csi-node-driver-daemonset.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "operator.fullname" . }}-csi-server + name: {{ include "operator.fullname" . }}-csi-node-driver labels: {{- include "operator.labels" . | nindent 4 }} spec: @@ -12,8 +12,8 @@ spec: template: metadata: annotations: - internal.stackable.tech/image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" - {{- with .Values.podAnnotations }} + internal.stackable.tech/image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- with .Values.csiNodeDriver.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: @@ -26,21 +26,21 @@ spec: # NOTE (@Techassi): Does it maybe make sense to have two different service accounts? serviceAccountName: {{ include "operator.fullname" . }}-serviceaccount securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml .Values.csiNodeDriver.podSecurityContext | nindent 8 }} containers: - - name: {{ include "operator.appname" . }}-csi-server + - name: csi-node-service securityContext: - {{- toYaml .Values.secretOperator.securityContext | nindent 12 }} - image: "{{ .Values.secretOperator.image.repository }}:{{ .Values.secretOperator.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.secretOperator.image.pullPolicy }} + {{- toYaml .Values.csiNodeDriver.nodeService.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} resources: - {{ .Values.secretOperator.resources | toYaml | nindent 12 }} + {{ .Values.csiNodeDriver.nodeService.resources | toYaml | nindent 12 }} # The arguments passed to the command being run in the container. The final command will # look like `secret-operator run csi-server [OPTIONS]`. The CSI server needs to run on # every Kubernetes cluster node and as such is deployed as a DaemonSet. args: - run - - csi-server + - csi-node-service env: # The following env vars are passed as clap (think CLI) arguments to the operator. # They are picked up by clap using the structs defied in the operator. @@ -50,7 +50,7 @@ spec: - name: CSI_ENDPOINT value: /csi/csi.sock - name: PRIVILEGED - value: {{ .Values.secretOperator.securityContext.privileged | quote }} + value: {{ .Values.csiNodeDriver.nodeService.securityContext.privileged | quote }} # Sometimes products need to know the operator image, e.g. the opa-bundle-builder OPA # sidecar uses the operator image. @@ -88,17 +88,18 @@ spec: - name: csi mountPath: /csi - name: mountpoint - mountPath: {{ .Values.kubeletDir }}/pods - {{- if .Values.secretOperator.securityContext.privileged }} + mountPath: {{ .Values.csiNodeDriver.kubeletDir }}/pods + {{- if .Values.csiNodeDriver.nodeService.securityContext.privileged }} mountPropagation: Bidirectional {{- end }} - name: tmp mountPath: /tmp + - name: external-provisioner - image: "{{ .Values.externalProvisioner.image.repository }}:{{ .Values.externalProvisioner.image.tag }}" - imagePullPolicy: {{ .Values.externalProvisioner.image.pullPolicy }} + image: "{{ .Values.csiNodeDriver.externalProvisioner.image.repository }}:{{ .Values.csiNodeDriver.externalProvisioner.image.tag }}" + imagePullPolicy: {{ .Values.csiNodeDriver.externalProvisioner.image.pullPolicy }} resources: - {{ .Values.externalProvisioner.resources | toYaml | nindent 12 }} + {{ .Values.csiNodeDriver.externalProvisioner.resources | toYaml | nindent 12 }} args: - --csi-address=/csi/csi.sock - --feature-gates=Topology=true @@ -106,14 +107,15 @@ spec: volumeMounts: - name: csi mountPath: /csi + - name: node-driver-registrar - image: "{{ .Values.nodeDriverRegistrar.image.repository }}:{{ .Values.nodeDriverRegistrar.image.tag }}" - imagePullPolicy: {{ .Values.nodeDriverRegistrar.image.pullPolicy }} + image: "{{ .Values.csiNodeDriver.nodeDriverRegistrar.image.repository }}:{{ .Values.csiNodeDriver.nodeDriverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.csiNodeDriver.nodeDriverRegistrar.image.pullPolicy }} resources: - {{ .Values.nodeDriverRegistrar.resources | toYaml | nindent 12 }} + {{ .Values.csiNodeDriver.nodeDriverRegistrar.resources | toYaml | nindent 12 }} args: - --csi-address=/csi/csi.sock - - --kubelet-registration-path={{ .Values.kubeletDir }}/plugins/secrets.stackable.tech/csi.sock + - --kubelet-registration-path={{ .Values.csiNodeDriver.kubeletDir }}/plugins/secrets.stackable.tech/csi.sock volumeMounts: - name: registration-sock mountPath: /registration @@ -124,27 +126,27 @@ spec: hostPath: # node-driver-registrar appends a driver-unique filename to this path to avoid conflicts # see https://github.com/stackabletech/secret-operator/issues/229 for why this path should not be too long - path: {{ .Values.kubeletDir }}/plugins_registry + path: {{ .Values.csiNodeDriver.kubeletDir }}/plugins_registry - name: csi hostPath: - path: {{ .Values.kubeletDir }}/plugins/secrets.stackable.tech/ + path: {{ .Values.csiNodeDriver.kubeletDir }}/plugins/secrets.stackable.tech/ - name: mountpoint hostPath: - path: {{ .Values.kubeletDir }}/pods/ + path: {{ .Values.csiNodeDriver.kubeletDir }}/pods/ - name: tmp emptyDir: {} - {{- with .Values.nodeSelector }} + {{- with .Values.csiNodeDriver.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with .Values.csiNodeDriver.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with .Values.csiNodeDriver.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.priorityClassName }} + {{- with .Values.csiNodeDriver.priorityClassName }} priorityClassName: {{ . }} {{- end }} diff --git a/deploy/helm/secret-operator/values.yaml b/deploy/helm/secret-operator/values.yaml index d90e7371..fbe32663 100644 --- a/deploy/helm/secret-operator/values.yaml +++ b/deploy/helm/secret-operator/values.yaml @@ -1,39 +1,20 @@ # Default values for secret-operator. --- +# Used by both the Controller Service and Node Service containers image: + repository: oci.stackable.tech/sdp/listener-operator + # tag: 0.0.0-dev + pullPolicy: IfNotPresent pullSecrets: [] -externalProvisioner: - image: - repository: oci.stackable.tech/sdp/sig-storage/csi-provisioner - tag: v5.3.0 - pullPolicy: IfNotPresent - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 100m - memory: 128Mi -nodeDriverRegistrar: - image: - repository: oci.stackable.tech/sdp/sig-storage/csi-node-driver-registrar - tag: v2.15.0 - pullPolicy: IfNotPresent - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 100m - memory: 128Mi +controllerService: + podAnnotations: {} + podSecurityContext: {} + # fsGroup: 2000 + nodeSelector: {} + tolerations: [] + affinity: {} -secretOperator: - image: - repository: oci.stackable.tech/sdp/secret-operator - # tag: 0.0.0-dev - pullPolicy: IfNotPresent - # Resources of the secret-operator container itself resources: limits: cpu: 100m @@ -45,10 +26,6 @@ secretOperator: securityContext: # secret-operator requires root permissions runAsUser: 0 - # It is strongly recommended to run secret-operator as a privileged container, since - # it enables additional protections for the secret contents. - # Unprivileged mode is EXPERIMENTAL and requires manual migration for an existing cluster. - privileged: true # capabilities: # drop: # - ALL @@ -56,6 +33,69 @@ secretOperator: # runAsNonRoot: true # runAsUser: 1000 +csiNodeDriver: + # Kubelet dir may vary in environments such as microk8s. + # See https://github.com/stackabletech/secret-operator/issues/229 + kubeletDir: /var/lib/kubelet + + podAnnotations: {} + podSecurityContext: {} + # fsGroup: 2000 + nodeSelector: {} + tolerations: [] + affinity: {} + + nodeService: + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + + securityContext: + # secret-operator requires root permissions + runAsUser: 0 + # It is strongly recommended to run secret-operator as a privileged container, since + # it enables additional protections for the secret contents. + # Unprivileged mode is EXPERIMENTAL and requires manual migration for an existing cluster. + privileged: true + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + externalProvisioner: + image: + repository: oci.stackable.tech/sdp/sig-storage/csi-provisioner + tag: v5.3.0 + pullPolicy: IfNotPresent + # NOTE (@Techassi): Support setting pullSecrets + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + + nodeDriverRegistrar: + image: + repository: oci.stackable.tech/sdp/sig-storage/csi-node-driver-registrar + tag: v2.15.0 + pullPolicy: IfNotPresent + # NOTE (@Techassi): Support setting pullSecrets + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + nameOverride: "" fullnameOverride: "" @@ -68,30 +108,11 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: "" -podAnnotations: {} - # Provide additional labels which get attached to all deployed resources labels: stackable.tech/vendor: Stackable -podSecurityContext: {} - # fsGroup: 2000 - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -# priorityClassName: ... - -# When running on a non-default Kubernetes cluster domain, the cluster domain can be configured here. -# See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details. -# kubernetesClusterDomain: my-cluster.local - -# Kubelet dir may vary in environments such as microk8s, see https://github.com/stackabletech/secret-operator/issues/229 -kubeletDir: /var/lib/kubelet - +# Customize default custom resources deployed by the operator secretClasses: tls: # The namespace that the TLS Certificate Authority is installed into. diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index e90c42d7..1ec67d61 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -59,14 +59,14 @@ struct SecretOperatorRun { #[derive(Debug, clap::Subcommand)] enum RunMode { /// Run the CSI server, one per Kubernetes cluster node. - CsiServer(CsiServerArguments), + CsiNodeService(CsiNodeServiceArguments), /// Run the controller, one per Kubernetes cluster. Controller, } #[derive(Debug, clap::Args)] -struct CsiServerArguments { +struct CsiNodeServiceArguments { #[arg(long, env)] csi_endpoint: PathBuf, @@ -137,7 +137,7 @@ async fn main() -> anyhow::Result<()> { .await?; match mode { - RunMode::CsiServer(CsiServerArguments { + RunMode::CsiNodeService(CsiNodeServiceArguments { csi_endpoint, privileged, }) => { From 678643034ebdb284c1cbe5f4c12bb43668bcfb4c Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 21 Oct 2025 12:50:57 +0200 Subject: [PATCH 5/5] chore: Adjust changelog entry --- CHANGELOG.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e60f8e0..c4c48d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,14 +20,13 @@ All notable changes to this project will be documented in this file. - The CSI server is deployed via a DaemonSet to be available on every node. - The controller is deployed via a Deployment with a single replica. - Version CRD structs and enums as v1alpha1 ([#636]). -- BREAKING: Rearrange values to be somewhat consistent with the listener-operator value changes ([#641]). - - `image.repository` has been moved to `secretOperator.image.repository`. - - `image.tag` has been moved to `secretOperator.image.tag`. - - `image.pullPolicy` has been moved to `secretOperator.image.pullPolicy`. - - `csiProvisioner` values have been moved to `externalProvisioner`. - - `csiNodeDriverRegistrar` values have been moved to `nodeDriverRegistrar`. - - `node.driver` values have been moved to `secretOperator`. - - `securityContext` values have been moved to `secretOperator.securityContext`. +- BREAKING: Rearrange values to be somewhat consistent with the listener-operator value changes ([#641], [#645]). + - `csiProvisioner` values have been moved to `csiNodeDriver.externalProvisioner`. + - `csiNodeDriverRegistrar` values have been moved to `csiNodeDriver.nodeDriverRegistrar`. + - `node.driver.resources` values have been split into `controllerService.resources` and `csiNodeDriver.nodeService.resources`. + - `securityContext` values have been split into `controllerService.securityContext` and `.csiNodeDriver.nodeService.securityContext`. + - `podAnnotations`, `podSecurityContext`, `nodeSelector`, `tolerations`, and `affinity` have been split into `controllerService` and `csiNodeDriver`. + - `kubeletDir` has been move to `csiNodeDriver.kubeletDir`. - Bump csi-node-driver-registrar to `v2.15.0` ([#642]). - Bump csi-provisioner to `v5.3.0` ([#643]).