test: add e2e tests for [Feature: WriteCertAndKeyInSeparateFiles]

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>
Azure · Nov 2, 2022 · 8adeaac · 8adeaac
1 parent c847884
commit 8adeaac
Show file tree

Hide file tree

Showing 11 changed files with 85 additions and 9 deletions.
diff --git a/.pipelines/templates/e2e-test-kind.yaml b/.pipelines/templates/e2e-test-kind.yaml
@@ -23,7 +23,7 @@ jobs:
           KIND_K8S_VERSION: v1.24.6
           IS_HELM_TEST: true
         kind_v1_25_2_helm:
-          KIND_K8S_VERSION: v1.25.0
+          KIND_K8S_VERSION: v1.25.2
           IS_HELM_TEST: true
         kind_v1_22_15_deployment_manifest:
           KIND_K8S_VERSION: v1.22.15

diff --git a/.pipelines/templates/e2e-test.yaml b/.pipelines/templates/e2e-test.yaml
@@ -72,3 +72,5 @@ steps:
         CI_KIND_CLUSTER: ${{ parameters.ciKindCluster }}
       ${{ if parameters.isArcTest }}:
         IS_ARC_TEST: ${{ parameters.isArcTest }}
+      ${{ if parameters.testReleasedVersion }}:
+        GINKGO_SKIP: WriteCertAndKeyInSeparateFiles
diff --git a/manifest_staging/charts/csi-secrets-store-provider-azure/README.md b/manifest_staging/charts/csi-secrets-store-provider-azure/README.md
@@ -144,3 +144,4 @@ The following table lists the configurable parameters of the csi-secrets-store-p
 | `rbac.install`                                                   | Install default service account                                                                                                                                                                       | true                                                                                             |
 | `rbac.pspEnabled`                                                | If `true`, create and use a restricted pod security policy for Secrets Store CSI Driver AKV provider pod(s)                                                                                           | false                                                                                            |
 | `constructPEMChain`                                              | Explicitly reconstruct the pem chain in the order: SERVER, INTERMEDIATE, ROOT                                                                                                                         | `true`                                                                                           |
+| `writeCertAndKeyInSeparateFiles`                                 | Write cert and key in separate files. The individual files will be named as <secret-name>.crt and <secret-name>.key. These files will be created in addition to the single file.                      | `false`                                                                                          |
diff --git a/...g/charts/csi-secrets-store-provider-azure/templates/provider-azure-installer-windows.yaml b/...g/charts/csi-secrets-store-provider-azure/templates/provider-azure-installer-windows.yaml
@@ -48,6 +48,9 @@ spec:
             - --healthz-port={{ .Values.windows.healthzPort }}
             - --healthz-path={{ .Values.windows.healthzPath }}
             - --healthz-timeout={{ .Values.windows.healthzTimeout }}
+            {{- if .Values.writeCertAndKeyInSeparateFiles }}
+            - --write-cert-and-key-in-separate-files={{ .Values.writeCertAndKeyInSeparateFiles }}
+            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.windows.healthzPath }}

diff --git a/...t_staging/charts/csi-secrets-store-provider-azure/templates/provider-azure-installer.yaml b/...t_staging/charts/csi-secrets-store-provider-azure/templates/provider-azure-installer.yaml
@@ -59,6 +59,9 @@ spec:
             - --healthz-port={{ .Values.linux.healthzPort }}
             - --healthz-path={{ .Values.linux.healthzPath }}
             - --healthz-timeout={{ .Values.linux.healthzTimeout }}
+            {{- if .Values.writeCertAndKeyInSeparateFiles }}
+            - --write-cert-and-key-in-separate-files={{ .Values.writeCertAndKeyInSeparateFiles }}
+            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.linux.healthzPath }}

diff --git a/manifest_staging/charts/csi-secrets-store-provider-azure/values.yaml b/manifest_staging/charts/csi-secrets-store-provider-azure/values.yaml
@@ -167,3 +167,6 @@ rbac:
 
 # explicitly reconstruct the pem chain in the order: SERVER, INTERMEDIATE, ROOT
 constructPEMChain: true
+
+# Write cert and key in separate files. The individual files will be named as <secret-name>.crt and <secret-name>.key. These files will be created in addition to the single file.
+writeCertAndKeyInSeparateFiles: false
diff --git a/test/e2e/certificates_test.go b/test/e2e/certificates_test.go
@@ -168,7 +168,7 @@ var _ = Describe("When fetching certificates and private key from Key Vault", fu
 		cmd = getPodExecCommand("cat /mnt/secrets-store/pemcert1-secret")
 		out, err := exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
 		Expect(err).To(BeNil())
-		certificates.ValidateCertBundle(out, pubKey, "test.domain.com")
+		certificates.ValidateCertBundle(out, pubKey, out, "test.domain.com")
 	})
 
 	It("should read pkcs12 cert, private and public key from pod", func() {
@@ -192,7 +192,7 @@ var _ = Describe("When fetching certificates and private key from Key Vault", fu
 		cmd = getPodExecCommand("cat /mnt/secrets-store/pkcs12cert1-secret")
 		out, err := exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
 		Expect(err).To(BeNil())
-		certificates.ValidateCertBundle(out, pubKey, "test.domain.com")
+		certificates.ValidateCertBundle(out, pubKey, out, "test.domain.com")
 
 		cmd = getPodExecCommand("cat /mnt/secrets-store/pkcs12cert1-secret-pfx")
 		out, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
@@ -204,7 +204,7 @@ var _ = Describe("When fetching certificates and private key from Key Vault", fu
 		pem, err := openssl.ParsePKCS12(string(pfxRaw), "")
 		Expect(err).To(BeNil())
 
-		certificates.ValidateCertBundle(pem, pubKey, "test.domain.com")
+		certificates.ValidateCertBundle(pem, pubKey, pem, "test.domain.com")
 	})
 
 	It("should read ecc cert, private and public key from pod", func() {
@@ -228,7 +228,7 @@ var _ = Describe("When fetching certificates and private key from Key Vault", fu
 		cmd = getPodExecCommand("cat /mnt/secrets-store/ecccert1-secret")
 		out, err := exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
 		Expect(err).To(BeNil())
-		certificates.ValidateCertBundle(out, pubKey, "")
+		certificates.ValidateCertBundle(out, pubKey, out, "")
 
 		cmd = getPodExecCommand("cat /mnt/secrets-store/ecccert1-secret-pfx")
 		out, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
@@ -240,6 +240,66 @@ var _ = Describe("When fetching certificates and private key from Key Vault", fu
 		pem, err := openssl.ParsePKCS12(string(pfxRaw), "")
 		Expect(err).To(BeNil())
 
-		certificates.ValidateCertBundle(pem, pubKey, "")
+		certificates.ValidateCertBundle(pem, pubKey, pem, "")
+	})
+
+	Describe("[Feature:WriteCertAndKeyInSeparateFiles] Writing certificates and private key in separate files", func() {
+		It("should write cert and key in separate files", func() {
+			pod.WaitFor(pod.WaitForInput{
+				Getter:         kubeClient,
+				KubeconfigPath: kubeconfigPath,
+				Config:         config,
+				PodName:        p.Name,
+				Namespace:      ns.Name,
+			})
+
+			// validate pemcert1
+			cmd := getPodExecCommand("cat /mnt/secrets-store/pemcert1-secret.crt")
+			cert, err := exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+			certificates.ValidateCert(cert, "test.domain.com")
+
+			cmd = getPodExecCommand("cat /mnt/secrets-store/pemcert1-pub-key")
+			pubKey, err := exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+
+			cmd = getPodExecCommand("cat /mnt/secrets-store/pemcert1-secret.key")
+			privKey, err := exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+
+			certificates.ValidateCertBundle(cert, pubKey, privKey, "test.domain.com")
+
+			// validate pkcs12cert1
+			cmd = getPodExecCommand("cat /mnt/secrets-store/pkcs12cert1-secret.crt")
+			cert, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+			certificates.ValidateCert(cert, "test.domain.com")
+
+			cmd = getPodExecCommand("cat /mnt/secrets-store/pkcs12cert1-pub-key")
+			pubKey, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+
+			cmd = getPodExecCommand("cat /mnt/secrets-store/pkcs12cert1-secret.key")
+			privKey, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+
+			certificates.ValidateCertBundle(cert, pubKey, privKey, "test.domain.com")
+
+			// validate ecccert1
+			cmd = getPodExecCommand("cat /mnt/secrets-store/ecccert1-secret.crt")
+			cert, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+			certificates.ValidateCert(cert, "test.domain.com")
+
+			cmd = getPodExecCommand("cat /mnt/secrets-store/ecccert1-pub-key")
+			pubKey, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+
+			cmd = getPodExecCommand("cat /mnt/secrets-store/ecccert1-secret.key")
+			privKey, err = exec.KubectlExec(kubeconfigPath, p.Name, p.Namespace, strings.Split(cmd, " "))
+			Expect(err).To(BeNil())
+
+			certificates.ValidateCertBundle(cert, pubKey, privKey, "test.domain.com")
+		})
 	})
 })
diff --git a/test/e2e/framework/certificates/certificates.go b/test/e2e/framework/certificates/certificates.go
@@ -46,12 +46,12 @@ func ValidateCert(certData, dnsName string) {
 
 // ValidateCertBundle validates the certificate, public key and private key returned by the provider match
 // and are usable
-func ValidateCertBundle(data, publicKey, dnsName string) {
+func ValidateCertBundle(data, publicKey, privKey, dnsName string) {
 	By(fmt.Sprintf("Ensuring certificate and private key is valid for dns name %s", dnsName))
 	certPEMBlock, err := getCert([]byte(data))
 	Expect(err).To(BeNil())
 
-	keyPEMBlock, err := getPrivateKey([]byte(data))
+	keyPEMBlock, err := getPrivateKey([]byte(privKey))
 	Expect(err).To(BeNil())
 
 	certs, err := X509KeyPair(certPEMBlock, keyPEMBlock, []byte(publicKey), []byte{})

diff --git a/test/e2e/framework/config.go b/test/e2e/framework/config.go
@@ -18,7 +18,7 @@ type Config struct {
 	KeyvaultName                      string `envconfig:"KEYVAULT_NAME"`
 	Registry                          string `envconfig:"REGISTRY" default:"mcr.microsoft.com/oss/azure/secrets-store"`
 	ImageName                         string `envconfig:"IMAGE_NAME" default:"provider-azure"`
-	ImageVersion                      string `envconfig:"IMAGE_VERSION" default:"v1.3.0`
+	ImageVersion                      string `envconfig:"IMAGE_VERSION" default:"v1.3.0"`
 	IsSoakTest                        bool   `envconfig:"IS_SOAK_TEST" default:"false"`
 	IsWindowsTest                     bool   `envconfig:"TEST_WINDOWS" default:"false"`
 	IsGPUTest                         bool   `envconfig:"TEST_GPU" default:"false"`

diff --git a/test/e2e/framework/deploy/deploy.go b/test/e2e/framework/deploy/deploy.go
@@ -102,6 +102,8 @@ func InstallManifest(kubeconfigPath string, config *framework.Config) {
 
 		// Configure higher log verbosity for debugging CI failures
 		ds.Spec.Template.Spec.Containers[0].Args = append(ds.Spec.Template.Spec.Containers[0].Args, "-v=5")
+		// Configure writeCertAndKeyInSeparateFiles to true as it's feature on top of default behavior
+		ds.Spec.Template.Spec.Containers[0].Args = append(ds.Spec.Template.Spec.Containers[0].Args, "--write-cert-and-key-in-separate-files=true")
 
 		updatedDS, err := yaml.Marshal(ds)
 		Expect(err).To(BeNil())

diff --git a/test/e2e/framework/helm/helm.go b/test/e2e/framework/helm/helm.go
@@ -55,6 +55,7 @@ func Install(input InstallInput) {
 		fmt.Sprintf("--set=logVerbosity=5"),
 		fmt.Sprintf("--set=linux.customUserAgent=csi-e2e"),
 		fmt.Sprintf("--set=windows.customUserAgent=csi-e2e"),
+		fmt.Sprintf("--set=writeCertAndKeyInSeparateFiles=true"),
 		"--dependency-update",
 		"--wait",
 		"--timeout=5m",
@@ -115,6 +116,7 @@ func Upgrade(input UpgradeInput) {
 		fmt.Sprintf("--set=logVerbosity=1"),
 		fmt.Sprintf("--set=linux.customUserAgent=csi-e2e"),
 		fmt.Sprintf("--set=windows.customUserAgent=csi-e2e"),
+		fmt.Sprintf("--set=writeCertAndKeyInSeparateFiles=true"),
 		"--wait",
 		"--timeout=5m",
 		"--debug",