Kube Gateway Policy Validation

.github/workflows/pr-kubernetes-tests.yaml

@@ -89,7 +89,7 @@ jobs:
           kubectl-version: 'v1.28.4'
           helm-version: 'v3.13.2'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '(^TestK8sGatewayIstio$$|^TestGlooGatewayEdgeGateway$$|^TestGlooctlIstioInjectEdgeApiGateway$$)'
+          go-test-run-regex: '(^TestK8sGatewayIstio$$|^TestGlooGatewayEdgeGateway$$|^TestGlooctlIstioInjectEdgeApiGateway$$|^TestK8sGatewayNoValidation$$)'
 
     steps:
     - id: auto-succeed-tests

changelog/v1.17.0-beta28/ggv2-validation.yaml

@@ -0,0 +1,6 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/solo-projects/issues/6063
+    resolvesIssue: true
+    description: |
+      Adds webhook validation for Gloo Gateway Policies (e.g. RouteOption and VirtualHostOption) when used with Kubernetes Gateway API

install/helm/gloo/templates/5-gateway-validation-webhook-configuration.yaml

@@ -24,6 +24,16 @@ webhooks:
       path: "/validation"
     caBundle: "" # update manually or use certgen job or cert-manager's ca-injector
   rules:
+{{- if .Values.kubeGateway.enabled }}
+  - operations: [ "CREATE", "UPDATE" ]
+    # RouteOption and VirtualHostOption DELETEs are not supported.
+    # Their validation is currently limited to usage as Kube Gateway API Policies
+    # and are hermetically validated for semantic correctness only. This means there
+    # is no validation needed for DELETEs, as a DELETE will never result be semantically invalid
+    apiGroups: ["gateway.solo.io"]
+    apiVersions: ["v1"]
+    resources: ["routeoptions", "virtualhostoptions"]
+{{- end }}{{/* if .Values.kubeGateway.enabled */}}
   - operations: {{ include "gloo.webhookvalidation.operationsForResource" (list "virtualservices" .Values.gateway.validation.webhook.skipDeleteValidationResources) }}
     apiGroups: ["gateway.solo.io"]
     apiVersions: ["v1"]

pkg/utils/kubeutils/kubectl/cli.go

@@ -87,20 +87,27 @@ func (c *Cli) Apply(ctx context.Context, content []byte, extraArgs ...string) er
 
 // ApplyFile applies the resources defined in a file, and returns an error if one occurred
 func (c *Cli) ApplyFile(ctx context.Context, fileName string, extraArgs ...string) error {
+	_, err := c.ApplyFileWithOutput(ctx, fileName, extraArgs...)
+	return err
+}
+
+// ApplyFileWithOutput applies the resources defined in a file,
+// if an error occurred, it will be returned along with the output of the command
+func (c *Cli) ApplyFileWithOutput(ctx context.Context, fileName string, extraArgs ...string) (string, error) {
 	applyArgs := append([]string{"apply", "-f", fileName}, extraArgs...)
 
 	fileInput, err := os.Open(fileName)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer func() {
 		_ = fileInput.Close()
 	}()
 
-	return c.Command(ctx, applyArgs...).
+	runErr := c.Command(ctx, applyArgs...).
 		WithStdin(fileInput).
-		Run().
-		Cause()
+		Run()
+	return runErr.OutputString(), runErr.Cause()
 }
 
 // Delete deletes the resources defined in the bytes, and returns an error if one occurred
@@ -114,20 +121,27 @@ func (c *Cli) Delete(ctx context.Context, content []byte, extraArgs ...string) e
 
 // DeleteFile deletes the resources defined in a file, and returns an error if one occurred
 func (c *Cli) DeleteFile(ctx context.Context, fileName string, extraArgs ...string) error {
+	_, err := c.DeleteFileWithOutput(ctx, fileName, extraArgs...)
+	return err
+}
+
+// DeleteFileWithOutput deletes the resources defined in a file,
+// if an error occurred, it will be returned along with the output of the command
+func (c *Cli) DeleteFileWithOutput(ctx context.Context, fileName string, extraArgs ...string) (string, error) {
 	applyArgs := append([]string{"delete", "-f", fileName}, extraArgs...)
 
 	fileInput, err := os.Open(fileName)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer func() {
 		_ = fileInput.Close()
 	}()
 
-	return c.Command(ctx, applyArgs...).
+	runErr := c.Command(ctx, applyArgs...).
 		WithStdin(fileInput).
-		Run().
-		Cause()
+		Run()
+	return runErr.OutputString(), runErr.Cause()
 }
 
 // DeleteFileSafe deletes the resources defined in a file, and returns an error if one occurred

projects/gateway/pkg/services/k8sadmission/validating_admission_webhook.go

@@ -99,9 +99,19 @@ type WebhookConfig struct {
 	alwaysAccept                  bool // accept all resources
 	readGatewaysFromAllNamespaces bool
 	webhookNamespace              string
+	kubeGatewayEnabled            bool
 }
 
-func NewWebhookConfig(ctx context.Context, validator validation.Validator, watchNamespaces []string, port int, serverCertPath, serverKeyPath string, alwaysAccept, readGatewaysFromAllNamespaces bool, webhookNamespace string) WebhookConfig {
+func NewWebhookConfig(
+	ctx context.Context,
+	validator validation.Validator,
+	watchNamespaces []string,
+	port int,
+	serverCertPath, serverKeyPath string,
+	alwaysAccept, readGatewaysFromAllNamespaces bool,
+	webhookNamespace string,
+	kubeGatewayEnabled bool,
+) WebhookConfig {
 	return WebhookConfig{
 		ctx:                           ctx,
 		validator:                     validator,
@@ -111,7 +121,9 @@ func NewWebhookConfig(ctx context.Context, validator validation.Validator, watch
 		serverKeyPath:                 serverKeyPath,
 		alwaysAccept:                  alwaysAccept,
 		readGatewaysFromAllNamespaces: readGatewaysFromAllNamespaces,
-		webhookNamespace:              webhookNamespace}
+		webhookNamespace:              webhookNamespace,
+		kubeGatewayEnabled:            kubeGatewayEnabled,
+	}
 }
 
 func NewGatewayValidatingWebhook(cfg WebhookConfig) (*http.Server, error) {
@@ -124,6 +136,7 @@ func NewGatewayValidatingWebhook(cfg WebhookConfig) (*http.Server, error) {
 	alwaysAccept := cfg.alwaysAccept
 	readGatewaysFromAllNamespaces := cfg.readGatewaysFromAllNamespaces
 	webhookNamespace := cfg.webhookNamespace
+	kubeGatewayEnabled := cfg.kubeGatewayEnabled
 
 	certProvider, err := NewCertificateProvider(serverCertPath, serverKeyPath, log.New(&debugLogger{ctx: ctx}, "validation-webhook-certificate-watcher", log.LstdFlags), ctx, 10*time.Second)
 	if err != nil {
@@ -137,6 +150,7 @@ func NewGatewayValidatingWebhook(cfg WebhookConfig) (*http.Server, error) {
 		alwaysAccept,
 		readGatewaysFromAllNamespaces,
 		webhookNamespace,
+		kubeGatewayEnabled,
 	)
 
 	mux := http.NewServeMux()
@@ -165,6 +179,7 @@ type gatewayValidationWebhook struct {
 	alwaysAccept                  bool                 // read only so no races
 	readGatewaysFromAllNamespaces bool                 // read only so no races
 	webhookNamespace              string               // read only so no races
+	kubeGatewayEnabled            bool                 // read only so no races
 }
 
 type AdmissionReviewWithProxies struct {
@@ -183,13 +198,24 @@ type AdmissionResponseWithProxies struct {
 	Proxies []*gloov1.Proxy `json:"proxies,omitempty"`
 }
 
-func NewGatewayValidationHandler(ctx context.Context, validator validation.Validator, watchNamespaces []string, alwaysAccept bool, readGatewaysFromAllNamespaces bool, webhookNamespace string) *gatewayValidationWebhook {
-	return &gatewayValidationWebhook{ctx: ctx,
+func NewGatewayValidationHandler(
+	ctx context.Context,
+	validator validation.Validator,
+	watchNamespaces []string,
+	alwaysAccept bool,
+	readGatewaysFromAllNamespaces bool,
+	webhookNamespace string,
+	kubeGatewayEnabled bool,
+) *gatewayValidationWebhook {
+	return &gatewayValidationWebhook{
+		ctx:                           ctx,
 		validator:                     validator,
 		watchNamespaces:               watchNamespaces,
 		alwaysAccept:                  alwaysAccept,
 		readGatewaysFromAllNamespaces: readGatewaysFromAllNamespaces,
-		webhookNamespace:              webhookNamespace}
+		webhookNamespace:              webhookNamespace,
+		kubeGatewayEnabled:            kubeGatewayEnabled,
+	}
 }
 
 func (wh *gatewayValidationWebhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -441,6 +467,11 @@ func (wh *gatewayValidationWebhook) validateGvk(ctx context.Context, gvk schema.
 	var reports *validation.Reports
 	newResourceFunc := gloosnapshot.ApiGvkToHashableResource[gvk]
 
+	// check to see if we should perform validation for this GVK
+	if !wh.shouldValidateGvk(gvk) {
+		return nil, nil
+	}
+
 	newResource := newResourceFunc()
 	oldResource := newResourceFunc()
 
@@ -475,6 +506,17 @@ func (wh *gatewayValidationWebhook) validateList(ctx context.Context, rawJson []
 	return reports, nil
 }
 
+func (wh *gatewayValidationWebhook) shouldValidateGvk(gvk schema.GroupVersionKind) bool {
+	if gvk == gwv1.RouteOptionGVK || gvk == gwv1.VirtualHostOptionGVK {
+		// only validate RouteOption and VirtualHostOption resources if K8s Gateway is enabled
+		return wh.kubeGatewayEnabled
+	}
+
+	// no other special considerations at this point, so continue with validation
+	return true
+}
+
+// shouldValidateResource determines if a resource should be validated AND populates `resource` (and `oldResource` if applicable) from the objects[s] in the `admissionRequest`
 func (wh *gatewayValidationWebhook) shouldValidateResource(ctx context.Context, admissionRequest *v1beta1.AdmissionRequest, resource, oldResource resources.HashableResource) (bool, error) {
 	logger := contextutils.LoggerFrom(ctx)
 

projects/gateway/pkg/services/k8sadmission/validating_admission_webhook_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/rotisserie/eris"
 	"github.com/solo-io/gloo/projects/gateway/pkg/validation"
 	validation2 "github.com/solo-io/gloo/projects/gloo/pkg/api/grpc/validation"
+	"github.com/solo-io/gloo/projects/gloo/pkg/api/v1/options/faultinjection"
+	"github.com/solo-io/gloo/projects/gloo/pkg/api/v1/options/headers"
 	"github.com/solo-io/gloo/projects/gloo/pkg/api/v1/options/static"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -37,11 +39,11 @@ import (
 
 var _ = Describe("ValidatingAdmissionWebhook", func() {
 
+	const errMsg = "didn't say the magic word"
 	var (
-		srv    *httptest.Server
-		mv     *mockValidator
-		wh     *gatewayValidationWebhook
-		errMsg = "didn't say the magic word"
+		srv *httptest.Server
+		mv  *mockValidator
+		wh  *gatewayValidationWebhook
 	)
 
 	BeforeEach(func() {
@@ -52,7 +54,6 @@ var _ = Describe("ValidatingAdmissionWebhook", func() {
 			validator:        mv,
 		}
 		srv = httptest.NewServer(wh)
-
 	})
 
 	setMockFunctions := func() {
@@ -110,6 +111,32 @@ var _ = Describe("ValidatingAdmissionWebhook", func() {
 
 	routeTable := &v1.RouteTable{Metadata: &core.Metadata{Namespace: "namespace", Name: "rt"}}
 
+	routeOption := &v1.RouteOption{
+		Metadata: &core.Metadata{
+			Name:      "policy",
+			Namespace: "default",
+		},
+		Options: &gloov1.RouteOptions{
+			Faults: &faultinjection.RouteFaults{
+				Abort: &faultinjection.RouteAbort{
+					Percentage: 4.19,
+					HttpStatus: 500,
+				},
+			},
+		},
+	}
+	vHostOption := &v1.VirtualHostOption{
+		Metadata: &core.Metadata{
+			Name:      "policy",
+			Namespace: "default",
+		},
+		Options: &gloov1.VirtualHostOptions{
+			HeaderManipulation: &headers.HeaderManipulation{
+				RequestHeadersToRemove: []string{"hello"},
+			},
+		},
+	}
+
 	DescribeTable("processes admission requests with auto-accept validator", func(crd crd.Crd, gvk schema.GroupVersionKind, op v1beta1.Operation, resourceOrRef interface{}) {
 		reviewRequest := makeReviewRequest(srv.URL, crd, gvk, op, resourceOrRef)
 		res, err := srv.Client().Do(reviewRequest)
@@ -431,6 +458,80 @@ var _ = Describe("ValidatingAdmissionWebhook", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})
+
+	Describe("Kube Gateway API Policy Validation", func() {
+		When("kubeGateway disabled", func() {
+			DescribeTable(
+				"always accepts",
+				func(fail bool, crd crd.Crd, gvk schema.GroupVersionKind, op v1beta1.Operation, resourceOrRef interface{}) {
+					if fail {
+						setMockFunctions()
+					}
+					reviewRequest := makeReviewRequest(srv.URL, crd, gvk, op, resourceOrRef)
+					res, err := srv.Client().Do(reviewRequest)
+					Expect(err).NotTo(HaveOccurred())
+
+					review, err := parseReviewResponse(res)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(review.Response).NotTo(BeNil())
+
+					Expect(review.Response.Allowed).To(BeTrue())
+					Expect(review.Proxies).To(BeEmpty())
+				},
+				Entry("RouteOption, CREATE, auto-accept = accepted", false, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Create, routeOption),
+				Entry("RouteOption, UPDATE, auto-accept = accepted", false, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Update, routeOption),
+				Entry("VirtualHostOption CREATE, auto-accept = accepted", false, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Create, vHostOption),
+				Entry("VirtualHostOption UPDATE, auto-accept = accepted", false, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Update, vHostOption),
+				Entry("RouteOption, CREATE, auto-fail = accepted", true, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Create, routeOption),
+				Entry("RouteOption, UPDATE, auto-fail = accepted", true, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Update, routeOption),
+				Entry("VirtualHostOption CREATE, auto-fail = accepted", true, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Create, vHostOption),
+				Entry("VirtualHostOption UPDATE, auto-fail = accepted", true, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Update, vHostOption),
+			)
+		})
+		When("kubeGateway enabled", func() {
+			JustBeforeEach(func() {
+				mv = &mockValidator{}
+				wh = &gatewayValidationWebhook{
+					webhookNamespace:   "namespace",
+					ctx:                context.TODO(),
+					validator:          mv,
+					kubeGatewayEnabled: true,
+				}
+				srv = httptest.NewServer(wh)
+			})
+
+			DescribeTable(
+				"webhook processes admission requests",
+				func(allowed bool, crd crd.Crd, gvk schema.GroupVersionKind, op v1beta1.Operation, resourceOrRef interface{}) {
+					if !allowed {
+						setMockFunctions()
+					}
+					reviewRequest := makeReviewRequest(srv.URL, crd, gvk, op, resourceOrRef)
+					res, err := srv.Client().Do(reviewRequest)
+					Expect(err).NotTo(HaveOccurred())
+
+					review, err := parseReviewResponse(res)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(review.Response).NotTo(BeNil())
+					if !allowed {
+						Expect(review.Response.Allowed).To(BeFalse())
+					} else {
+						Expect(review.Response.Allowed).To(BeTrue())
+					}
+					Expect(review.Proxies).To(BeEmpty())
+				},
+				Entry("RouteOption, CREATE, auto-accept = accepted", true, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Create, routeOption),
+				Entry("VirtualHostOption CREATE, auto-accept = accepted", true, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Create, vHostOption),
+				Entry("RouteOption, CREATE, auto-fail = fail", false, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Create, routeOption),
+				Entry("VirtualHostOption CREATE, auto-fail = fail", false, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Create, vHostOption),
+				// TODO(Law): add UPDATE tests here, need to handle oldObject similar to the status update tests
+				// Entry("RouteOption, UPDATE, auto-accept = accepted", true, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Update, routeOption),
+				// Entry("VirtualHostOption UPDATE, auto-accept = accepted", true, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Update, vHostOption),
+				// Entry("RouteOption, UPDATE, auto-fail = fail", false, v1.RouteOptionCrd, v1.RouteOptionCrd.GroupVersionKind(), v1beta1.Update, routeOption),
+				// Entry("VirtualHostOption UPDATE, auto-fail = fail", false, v1.VirtualHostOptionCrd, v1.VirtualHostOptionCrd.GroupVersionKind(), v1beta1.Update, vHostOption),
+			)
+		})
+	})
 })
 
 func makeReviewRequest(url string, crd crd.Crd, gvk schema.GroupVersionKind, operation v1beta1.Operation, resource interface{}) *http.Request {