[1.16.x] secret delete and robustness flake backport (#9263)

* Robustness test flake fix (#9252)

* update `AfterAll` cleanup

* Adding changelog file to new location

* Deleting changelog file from old location

* better update

* PR feedback

* Update util.go

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>
Co-authored-by: Bernie Birnbaum <bewebi@earthlink.net>

* logging and verifyGlooValidationWorks (#9250)

* logging and verifyGlooValidationWorks

* Flake Debugging

* PR feedback

* Update delete-secret-flake.yaml

* Update util.go

* Adding changelog file to new location

* Deleting changelog file from old location

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>

* update

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: Bernie Birnbaum <bewebi@earthlink.net>

changelog/v1.16.9/delete-secret-and-robustness-flake.yaml

@@ -0,0 +1,13 @@
+changelog:
+  - type: NON_USER_FACING
+    resolvesIssue: false
+    issueLink: https://github.com/solo-io/gloo/issues/8826
+    description: >-
+      - Added `verifyGlooValidationWorks` to the beginning of flaky secret delete test to validate that validation is ready before starting the test
+      - Fixed an issue where the kube/envoy artifacts were not being dumped when the test failed
+  - type: NON_USER_FACING
+    resolvesIssue: false
+    issueLink: https://github.com/solo-io/gloo/issues/9251
+    description: >-
+      The flake seems to be occuring due to improper cleanup of setting the validatingAdmissionEebhook failure policy to `Ignore`` after setting it to `Fail`` for a set of tests
+      This PR fixes the flake by setting the failure policy to `Ignore` in the `AfterAll`

test/helpers/kube_dump.go

@@ -25,8 +25,8 @@ var (
 // Look at `KubeDumpOnFail` && `EnvoyDumpOnFail` for more details
 func StandardGlooDumpOnFail(out io.Writer, namespaces ...string) func() {
 	return func() {
-		KubeDumpOnFail(out, namespaces...)
-		EnvoyDumpOnFail(out, namespaces...)
+		KubeDumpOnFail(out, namespaces...)()
+		EnvoyDumpOnFail(out, namespaces...)()
 	}
 }
 

test/kube2e/gateway/gateway_test.go

@@ -2114,9 +2114,17 @@ spec:
 				// There are times when the VirtualService + Proxy do not update Status with the error when deleting the referenced Secret, therefore the validation error doesn't occur.
 				// It isn't until later - either a few minutes and/or after forcing an update by updating the VS - that the error status appears.
 				// The reason is still unknown, so we retry on flakes in the meantime.
-				It("should act as expected with secret validation", FlakeAttempts(3), func() {
+				It("should act as expected with secret validation", func() {
+					verifyGlooValidationWorks()
+
+					By("waiting for the modified VS to be accepted")
+					helpers.EventuallyResourceAccepted(func() (resources.InputResource, error) {
+						return resourceClientset.VirtualServiceClient().Read(testHelper.InstallNamespace, testRunnerVs.GetMetadata().GetName(), clients.ReadOpts{Ctx: ctx})
+					})
+
 					By("failing to delete a secret that is in use")
 					err := resourceClientset.KubeClients().CoreV1().Secrets(testHelper.InstallNamespace).Delete(ctx, secretName, metav1.DeleteOptions{})
+
 					Expect(err).To(HaveOccurred())
 					Expect(err.Error()).To(matchers2.ContainSubstrings([]string{"admission webhook", "SSL secret not found", secretName}))
 
@@ -2216,8 +2224,9 @@ spec:
 			)
 
 			var (
-				invalidUpstreamYaml string
-				vsYaml              string
+				invalidUpstreamYaml      string
+				vsYaml                   string
+				pretestFailurePolicyType admissionregv1.FailurePolicyType
 			)
 
 			// Before these secret deletion tests, set the failure policy to Fail and setup the resources with warnings
@@ -2255,6 +2264,9 @@ spec:
             name: my-us
             namespace:  ` + testHelper.InstallNamespace
 
+				// Store the current failure policy to restore after the tests
+				pretestFailurePolicyType = *kube2e.GetFailurePolicy(ctx, "gloo-gateway-validation-webhook-"+testHelper.InstallNamespace)
+
 				kube2e.UpdateFailurePolicy(ctx, "gloo-gateway-validation-webhook-"+testHelper.InstallNamespace, admissionregv1.Fail)
 				// Allow warnings during setup so that we can install the resources
 				kube2e.UpdateAllowWarningsSetting(ctx, true, testHelper.InstallNamespace)
@@ -2276,7 +2288,7 @@ spec:
 			})
 
 			AfterAll(func() {
-				kube2e.UpdateFailurePolicy(ctx, "gloo-gateway-validation-webhook-"+testHelper.InstallNamespace, admissionregv1.Fail)
+				kube2e.UpdateFailurePolicy(ctx, "gloo-gateway-validation-webhook-"+testHelper.InstallNamespace, pretestFailurePolicyType)
 				err := install.KubectlDelete([]byte(invalidUpstreamYaml))
 				Expect(err).NotTo(HaveOccurred())
 				err = install.KubectlDelete([]byte(vsYaml))

test/kube2e/util.go

@@ -299,12 +299,28 @@ const SimpleTestRunnerHttpResponseArm = `<!DOCTYPE html PUBLIC "-//W3C//DTD HTML
 </body>
 </html>`
 
+func GetFailurePolicy(ctx context.Context, webhookName string) *admissionregv1.FailurePolicyType {
+	cfg := GetValidatingWebhookWithOffset(ctx, 2, webhookName)
+	ExpectWithOffset(1, cfg.Webhooks).To(HaveLen(1))
+	return cfg.Webhooks[0].FailurePolicy
+}
+
 func UpdateFailurePolicy(ctx context.Context, webhookName string, failurePolicy admissionregv1.FailurePolicyType) {
 	kubeClient := clienthelpers.MustKubeClient()
-	cfg, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, webhookName, metav1.GetOptions{})
-	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+	cfg := GetValidatingWebhookWithOffset(ctx, 2, webhookName)
+	ExpectWithOffset(1, cfg.Webhooks).To(HaveLen(1))
 	cfg.Webhooks[0].FailurePolicy = &failurePolicy
 
-	_, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(ctx, cfg, metav1.UpdateOptions{})
+	_, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(ctx, cfg, metav1.UpdateOptions{})
 	ExpectWithOffset(1, err).NotTo(HaveOccurred())
 }
+func GetValidatingWebhook(ctx context.Context, webhookName string) *admissionregv1.ValidatingWebhookConfiguration {
+	return GetValidatingWebhookWithOffset(ctx, 1, webhookName)
+}
+
+func GetValidatingWebhookWithOffset(ctx context.Context, offset int, webhookName string) *admissionregv1.ValidatingWebhookConfiguration {
+	kubeClient := clienthelpers.MustKubeClient()
+	cfg, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, webhookName, metav1.GetOptions{})
+	ExpectWithOffset(offset, err).NotTo(HaveOccurred())
+	return cfg
+}