feat(*) allow ca.crt to be in separate k8s secret (#3638)

* feat(*): allow ca.crt to be in separate k8s secret

This feature adds option to provide ca.crt in separate k8s secret
than tls.key/tls.crt, which allows to provide CA and certificates
using for example cert-manager

+ two small stylistic changes in helm chart's values.yaml file
  suggested by my IDE

Signed-off-by: Bart Smykla <bartek@smykla.com>

app/kumactl/cmd/completion/testdata/bash.golden

@@ -2842,6 +2842,10 @@ _kumactl_install_control-plane()
     two_word_flags+=("--tls-general-ca-bundle")
     local_nonpersistent_flags+=("--tls-general-ca-bundle")
     local_nonpersistent_flags+=("--tls-general-ca-bundle=")
+    flags+=("--tls-general-ca-secret=")
+    two_word_flags+=("--tls-general-ca-secret")
+    local_nonpersistent_flags+=("--tls-general-ca-secret")
+    local_nonpersistent_flags+=("--tls-general-ca-secret=")
     flags+=("--tls-general-secret=")
     two_word_flags+=("--tls-general-secret")
     local_nonpersistent_flags+=("--tls-general-secret")

app/kumactl/cmd/install/context/install_control_plane_context.go

@@ -15,6 +15,7 @@ type InstallControlPlaneArgs struct {
 	ControlPlane_image_tag                       string            `helm:"controlPlane.image.tag"`
 	ControlPlane_service_name                    string            `helm:"controlPlane.service.name"`
 	ControlPlane_tls_general_secret              string            `helm:"controlPlane.tls.general.secretName"`
+	ControlPlane_tls_general_ca_secret           string            `helm:"controlPlane.tls.general.caSecretName"`
 	ControlPlane_tls_general_caBundle            string            `helm:"controlPlane.tls.general.caBundle"`
 	ControlPlane_tls_apiServer_secret            string            `helm:"controlPlane.tls.apiServer.secretName"`
 	ControlPlane_tls_apiServer_clientCertsSecret string            `helm:"controlPlane.tls.apiServer.clientCertsSecretName"`

app/kumactl/cmd/install/install_control_plane.go

@@ -112,11 +112,12 @@ This command requires that the KUBECONFIG environment is set`,
 	cmd.Flags().StringVar(&args.ControlPlane_image_repository, "control-plane-repository", args.ControlPlane_image_repository, "repository for the image of the Kuma Control Plane component")
 	cmd.Flags().StringVar(&args.ControlPlane_image_tag, "control-plane-version", args.ControlPlane_image_tag, "version of the image of the Kuma Control Plane component")
 	cmd.Flags().StringVar(&args.ControlPlane_service_name, "control-plane-service-name", args.ControlPlane_service_name, "Service name of the Kuma Control Plane")
-	cmd.Flags().StringVar(&args.ControlPlane_tls_general_secret, "tls-general-secret", args.ControlPlane_tls_general_secret, "Secret that contains tls.crt, key.crt and ca.crt for protecting Kuma in-cluster communication")
+	cmd.Flags().StringVar(&args.ControlPlane_tls_general_secret, "tls-general-secret", args.ControlPlane_tls_general_secret, "Secret that contains tls.crt, tls.key [and ca.crt when no --tls-general-ca-secret specified] for protecting Kuma in-cluster communication")
+	cmd.Flags().StringVar(&args.ControlPlane_tls_general_ca_secret, "tls-general-ca-secret", args.ControlPlane_tls_general_ca_secret, "Secret that contains ca.crt that was used to sign cert for protecting Kuma in-cluster communication (ca.crt present in this secret have precedence over the one provided in --tls-general-secret)")
 	cmd.Flags().StringVar(&args.ControlPlane_tls_general_caBundle, "tls-general-ca-bundle", args.ControlPlane_tls_general_secret, "Base64 encoded CA certificate (the same as in controlPlane.tls.general.secret#ca.crt)")
-	cmd.Flags().StringVar(&args.ControlPlane_tls_apiServer_secret, "tls-api-server-secret", args.ControlPlane_tls_apiServer_secret, "Secret that contains tls.crt, key.crt for protecting Kuma API on HTTPS")
+	cmd.Flags().StringVar(&args.ControlPlane_tls_apiServer_secret, "tls-api-server-secret", args.ControlPlane_tls_apiServer_secret, "Secret that contains tls.crt, tls.key for protecting Kuma API on HTTPS")
 	cmd.Flags().StringVar(&args.ControlPlane_tls_apiServer_clientCertsSecret, "tls-api-server-client-certs-secret", args.ControlPlane_tls_apiServer_clientCertsSecret, "Secret that contains list of .pem certificates that can access admin endpoints of Kuma API on HTTPS")
-	cmd.Flags().StringVar(&args.ControlPlane_tls_kdsGlobalServer_secret, "tls-kds-global-server-secret", args.ControlPlane_tls_kdsGlobalServer_secret, "Secret that contains tls.crt, key.crt for protecting cross cluster communication")
+	cmd.Flags().StringVar(&args.ControlPlane_tls_kdsGlobalServer_secret, "tls-kds-global-server-secret", args.ControlPlane_tls_kdsGlobalServer_secret, "Secret that contains tls.crt, tls.key for protecting cross cluster communication")
 	cmd.Flags().StringVar(&args.ControlPlane_tls_kdsZoneClient_secret, "tls-kds-zone-client-secret", args.ControlPlane_tls_kdsZoneClient_secret, "Secret that contains ca.crt which was used to sign KDS Global server. Used for CP verification")
 	cmd.Flags().StringVar(&args.ControlPlane_injectorFailurePolicy, "injector-failure-policy", args.ControlPlane_injectorFailurePolicy, "failure policy of the mutating web hook implemented by the Kuma Injector component")
 	cmd.Flags().StringToStringVar(&args.ControlPlane_envVars, "env-var", args.ControlPlane_envVars, "environment variables that will be passed to the control plane")

app/kumactl/cmd/install/install_control_plane_test.go

@@ -57,7 +57,15 @@ var _ = Describe("kumactl install control-plane", func() {
 
 			// given
 			rootCmd := cmd.NewRootCmd(rootCtx)
-			rootCmd.SetArgs(append([]string{"install", "control-plane", "--tls-general-secret", "general-tls-secret", "--tls-general-ca-bundle", "XYZ"}, given.extraArgs...))
+			rootCmd.SetArgs(append(
+				[]string{
+					"install",
+					"control-plane",
+					"--tls-general-secret", "general-tls-secret",
+					"--tls-general-ca-bundle", "XYZ",
+				},
+				given.extraArgs...,
+			))
 			rootCmd.SetOut(stdout)
 			rootCmd.SetErr(stderr)
 
@@ -101,6 +109,7 @@ var _ = Describe("kumactl install control-plane", func() {
 				"--tls-api-server-client-certs-secret", "api-server-client-secret",
 				"--tls-kds-global-server-secret", "kds-global-secret",
 				"--tls-kds-zone-client-secret", "kds-ca-secret",
+				"--tls-general-ca-secret", "general-tls-secret-ca",
 				"--mode", "zone",
 				"--kds-global-address", "grpcs://192.168.0.1:5685",
 				"--zone", "zone-1",

app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml

@@ -1463,7 +1463,16 @@ spec:
               memory: 256Mi
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: kuma-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane

app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml

@@ -1288,7 +1288,16 @@ spec:
               memory: 256Mi
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: kuma-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane

app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml

@@ -1295,7 +1295,16 @@ spec:
               memory: 256Mi
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: kuma-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane

app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml

@@ -1288,7 +1288,16 @@ spec:
               memory: 256Mi
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: kuma-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane

app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml

@@ -1308,7 +1308,16 @@ spec:
               memory: 256Mi
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+            - name: general-tls-cert-ca
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: kuma-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane
@@ -1329,6 +1338,9 @@ spec:
         - name: general-tls-cert
           secret:
             secretName: general-tls-secret
+        - name: general-tls-cert-ca
+          secret:
+            secretName: general-tls-secret-ca
         - name: api-server-tls-cert
           secret:
             secretName: api-server-secret

app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml

@@ -1317,7 +1317,16 @@ spec:
               memory: 256Mi
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: kuma-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane

app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml

@@ -1296,7 +1296,16 @@ spec:
               memory: 256Mi
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: kuma-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane

deployments/charts/kuma/README.md

@@ -12,7 +12,7 @@ A Helm chart for the Kuma Control Plane
 |-----|------|---------|-------------|
 | global.image.registry | string | `"docker.io/kumahq"` | Default registry for all Kuma Images |
 | global.image.tag | string | `nil` | The default tag for all Kuma images, which itself defaults to .Chart.AppVersion |
-| patchSystemNamespace | bool | `true` | Whether or not to patch the target namespace with the system label |
+| patchSystemNamespace | bool | `true` | Whether to patch the target namespace with the system label |
 | installCrdsOnUpgrade | object | `{"enabled":true,"imagePullSecrets":[]}` | Whether ot not install new CRDs before upgrade  (if any were introduced    with the new version of Kuma) |
 | controlPlane.logLevel | string | `"info"` | Kuma CP log level: one of off,info,debug |
 | controlPlane.mode | string | `"standalone"` | Kuma CP modes: one of standalone,zone,global |
@@ -35,13 +35,14 @@ A Helm chart for the Kuma Control Plane
 | controlPlane.globalZoneSyncService.loadBalancerIP | string | `nil` | Optionally specify IP to be used by cloud provider when configuring load balancer |
 | controlPlane.globalZoneSyncService.annotations | object | `{}` | Additional annotations to put on the Global Zone Sync Service |
 | controlPlane.globalZoneSyncService.port | int | `5685` | Port on which Global Zone Sync Service is exposed |
-| controlPlane.defaults.skipMeshCreation | bool | `false` | Whether or not to skip creating the default Mesh |
+| controlPlane.defaults.skipMeshCreation | bool | `false` | Whether to skip creating the default Mesh |
 | controlPlane.resources | string | `nil` | Optionally override the resource spec |
-| controlPlane.tls.general.secretName | string | `""` | Secret that contains tls.crt, key.crt and ca.crt for protecting Kuma in-cluster communication |
+| controlPlane.tls.general.secretName | string | `""` | Secret that contains tls.crt, tls.key [and ca.crt when no controlPlane.tls.general.caSecretName specified] for protecting Kuma in-cluster communication |
+| controlPlane.tls.general.caSecretName | string | `""` | Secret that contains ca.crt that was used to sign cert for protecting Kuma in-cluster communication (ca.crt present in this secret have precedence over the one provided in the controlPlane.tls.general.secretName) |
 | controlPlane.tls.general.caBundle | string | `""` | Base64 encoded CA certificate (the same as in controlPlane.tls.general.secret#ca.crt) |
-| controlPlane.tls.apiServer.secretName | string | `""` | Secret that contains tls.crt, key.crt for protecting Kuma API on HTTPS |
+| controlPlane.tls.apiServer.secretName | string | `""` | Secret that contains tls.crt, tls.key for protecting Kuma API on HTTPS |
 | controlPlane.tls.apiServer.clientCertsSecretName | string | `""` | Secret that contains list of .pem certificates that can access admin endpoints of Kuma API on HTTPS |
-| controlPlane.tls.kdsGlobalServer.secretName | string | `""` | Secret that contains tls.crt, key.crt for protecting cross cluster communication |
+| controlPlane.tls.kdsGlobalServer.secretName | string | `""` | Secret that contains tls.crt, tls.key for protecting cross cluster communication |
 | controlPlane.tls.kdsZoneClient.secretName | string | `""` | Secret that contains ca.crt which was used to sign KDS Global server. Used for CP verification |
 | controlPlane.image.pullPolicy | string | `"IfNotPresent"` | Kuma CP ImagePullPolicy |
 | controlPlane.image.repository | string | `"kuma-cp"` | Kuma CP image repository |

deployments/charts/kuma/templates/cp-deployment.yaml

@@ -110,7 +110,20 @@ spec:
           {{- end }}
           volumeMounts:
             - name: general-tls-cert
-              mountPath: /var/run/secrets/kuma.io/tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
+              subPath: tls.crt
+              readOnly: true
+            - name: general-tls-cert
+              mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
+              subPath: tls.key
+              readOnly: true
+          {{- if .Values.controlPlane.tls.general.caSecretName }}
+            - name: general-tls-cert-ca
+          {{- else}}
+            - name: general-tls-cert
+          {{- end }}
+              mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
+              subPath: ca.crt
               readOnly: true
             - name: {{ include "kuma.name" . }}-control-plane-config
               mountPath: /etc/kuma.io/kuma-control-plane
@@ -145,6 +158,11 @@ spec:
           secret:
             secretName: {{ include "kuma.name" . }}-tls-cert
         {{- end }}
+        {{- if .Values.controlPlane.tls.general.caSecretName }}
+        - name: general-tls-cert-ca
+          secret:
+            secretName: {{ .Values.controlPlane.tls.general.caSecretName }}
+        {{- end }}
         {{- if .Values.controlPlane.tls.apiServer.secretName }}
         - name: api-server-tls-cert
           secret:

deployments/charts/kuma/values.yaml

@@ -5,7 +5,7 @@ global:
     # -- The default tag for all Kuma images, which itself defaults to .Chart.AppVersion
     tag:
 
-# -- Whether or not to patch the target namespace with the system label
+# -- Whether to patch the target namespace with the system label
 patchSystemNamespace: true
 
 # -- Whether ot not install new CRDs before upgrade  (if any were introduced
@@ -84,7 +84,7 @@ controlPlane:
     port: 5685
 
   defaults:
-    # -- Whether or not to skip creating the default Mesh
+    # -- Whether to skip creating the default Mesh
     skipMeshCreation: false
 
   # -- Optionally override the resource spec
@@ -97,17 +97,23 @@ controlPlane:
   # TLS for various servers
   tls:
     general:
-      # -- Secret that contains tls.crt, key.crt and ca.crt for protecting Kuma in-cluster communication
+      # -- Secret that contains tls.crt, tls.key [and ca.crt when no
+      # controlPlane.tls.general.caSecretName specified] for protecting
+      # Kuma in-cluster communication
       secretName: ""
+      # -- Secret that contains ca.crt that was used to sign cert for protecting
+      # Kuma in-cluster communication (ca.crt present in this secret
+      # have precedence over the one provided in the controlPlane.tls.general.secretName)
+      caSecretName: ""
       # -- Base64 encoded CA certificate (the same as in controlPlane.tls.general.secret#ca.crt)
       caBundle: ""
     apiServer:
-      # -- Secret that contains tls.crt, key.crt for protecting Kuma API on HTTPS
+      # -- Secret that contains tls.crt, tls.key for protecting Kuma API on HTTPS
       secretName: ""
       # -- Secret that contains list of .pem certificates that can access admin endpoints of Kuma API on HTTPS
       clientCertsSecretName: ""
     kdsGlobalServer:
-      # -- Secret that contains tls.crt, key.crt for protecting cross cluster communication
+      # -- Secret that contains tls.crt, tls.key for protecting cross cluster communication
       secretName: ""
     kdsZoneClient:
       # -- Secret that contains ca.crt which was used to sign KDS Global server. Used for CP verification

docs/cmd/kumactl/kumactl_install_control-plane.md

@@ -43,10 +43,11 @@ kumactl install control-plane [flags]
       --mode string                                 kuma cp modes: one of standalone|zone|global (default "standalone")
       --namespace string                            namespace to install Kuma Control Plane to (default "kuma-system")
       --tls-api-server-client-certs-secret string   Secret that contains list of .pem certificates that can access admin endpoints of Kuma API on HTTPS
-      --tls-api-server-secret string                Secret that contains tls.crt, key.crt for protecting Kuma API on HTTPS
+      --tls-api-server-secret string                Secret that contains tls.crt, tls.key for protecting Kuma API on HTTPS
       --tls-general-ca-bundle string                Base64 encoded CA certificate (the same as in controlPlane.tls.general.secret#ca.crt)
-      --tls-general-secret string                   Secret that contains tls.crt, key.crt and ca.crt for protecting Kuma in-cluster communication
-      --tls-kds-global-server-secret string         Secret that contains tls.crt, key.crt for protecting cross cluster communication
+      --tls-general-ca-secret string                Secret that contains ca.crt that was used to sign cert for protecting Kuma in-cluster communication (ca.crt present in this secret have precedence over the one provided in --tls-general-secret)
+      --tls-general-secret string                   Secret that contains tls.crt, tls.key [and ca.crt when no --tls-general-ca-secret specified] for protecting Kuma in-cluster communication
+      --tls-kds-global-server-secret string         Secret that contains tls.crt, tls.key for protecting cross cluster communication
       --tls-kds-zone-client-secret string           Secret that contains ca.crt which was used to sign KDS Global server. Used for CP verification
       --use-node-port                               use NodePort instead of LoadBalancer
       --version string                              version of Kuma Control Plane components