feat: Add Constraint schema validation testing (open-policy-agent#2092)

* feat: Add Constraint schema validation testing

Add feature to "gator verify" which allows testing the schema of
Constraints.

Fixes: open-policy-agent#1989
Signed-off-by: Will Beason <willbeason@google.com>

* fix lint errors and comments

Signed-off-by: Max Smythe <smythe@google.com>

* tweak error message

Signed-off-by: Max Smythe <smythe@google.com>

Co-authored-by: Will Beason <willbeason@google.com>
Co-authored-by: Max Smythe <smythe@google.com>
Co-authored-by: Rita Zhang <rita.z.zhang@gmail.com>
Signed-off-by: Christoph Fraundorfer <christoph.fraundorfer@allianzdirect.de>

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
 	github.com/open-policy-agent/cert-controller v0.3.0
-	github.com/open-policy-agent/frameworks/constraint v0.0.0-20220527154834-b482a682709a
+	github.com/open-policy-agent/frameworks/constraint v0.0.0-20220603201627-e8d94b2fcf7a
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.2
 	github.com/spf13/cobra v1.4.0

go.sum

@@ -820,8 +820,8 @@ github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
 github.com/open-policy-agent/cert-controller v0.3.0 h1:9eUgN3yYMZsfyW7qdW8+CX9YZCUb5R5JfRTj0cqaSVg=
 github.com/open-policy-agent/cert-controller v0.3.0/go.mod h1:uOQW+2tMU51vSxy1Yt162oVUTMdqLuotC0aObQxrh6k=
-github.com/open-policy-agent/frameworks/constraint v0.0.0-20220527154834-b482a682709a h1:li3My6X1UgTahY+UZOi3HYQw01tZqjjn+Fh0RBzXu4k=
-github.com/open-policy-agent/frameworks/constraint v0.0.0-20220527154834-b482a682709a/go.mod h1:8GTSaIKto8bel+l15v1Zwm9qXF3LCgdHatn7+rj5Jhw=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20220603201627-e8d94b2fcf7a h1:YFthks7sIYmiOsElAwUZ48VdN9LI7ekF/vV4wYeR9mk=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20220603201627-e8d94b2fcf7a/go.mod h1:8GTSaIKto8bel+l15v1Zwm9qXF3LCgdHatn7+rj5Jhw=
 github.com/open-policy-agent/opa v0.40.0 h1:z/eg0ff3O1y6ovxpbL7xv+NHSwi8rVA7993sLv5Owac=
 github.com/open-policy-agent/opa v0.40.0/go.mod h1:UQqv8nJ1njs2+Od1lrPFzUAApdj22ABxTO35+Vpsjz4=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=

pkg/gator/errors.go

@@ -38,4 +38,7 @@ var (
 	// ErrConvertingTemplate means we were able to parse a template, but not convert
 	// it into the version-independent format.
 	ErrConvertingTemplate = errors.New("unable to convert template")
+	// ErrValidConstraint occurs when a test's configuration signals an expectation
+	// that a constraint should fail validation but no validation error is raised.
+	ErrValidConstraint = errors.New("constraint should have failed schema validation")
 )

pkg/gator/fixtures/fixtures.go

@@ -392,4 +392,58 @@ spec:
   selector:
     key: value
 `
+
+	TemplateRequiredLabel = `
+apiVersion: templates.gatekeeper.sh/v1
+kind: ConstraintTemplate
+metadata:
+  name: k8srequiredlabels
+spec:
+  crd:
+    spec:
+      names:
+        kind: K8sRequiredLabels
+      validation:
+        # Schema for the parameters field
+        openAPIV3Schema:
+          type: object
+          properties:
+            labels:
+              type: array
+              items:
+                type: string
+  targets:
+    - target: admission.k8s.gatekeeper.sh
+      rego: |
+        package k8srequiredlabels
+
+        violation[{"msg": msg, "details": {"missing_labels": missing}}] {
+          provided := {label | input.review.object.metadata.labels[label]}
+          required := {label | label := input.parameters.labels[_]}
+          missing := required - provided
+          count(missing) > 0
+          ns := [n | data.inventory.cluster.v1.Namespace[n]]
+          msg := sprintf("I can grab namespaces... %v ... and dump the inventory... %v", [ns, data.inventory])
+        }
+`
+
+	ConstraintRequireLabelInvalid = `
+apiVersion: constraints.gatekeeper.sh/v1beta1
+kind: K8sRequiredLabels
+metadata:
+  name: required-labels
+spec:
+  parameters:
+    labels: "abc"
+`
+
+	ConstraintRequireLabelValid = `
+apiVersion: constraints.gatekeeper.sh/v1beta1
+kind: K8sRequiredLabels
+metadata:
+  name: required-labels
+spec:
+  parameters:
+    labels: ["abc"]
+`
 )

pkg/gator/runner.go

@@ -2,11 +2,13 @@ package gator
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/fs"
 	"path"
 	"time"
 
+	"github.com/open-policy-agent/frameworks/constraint/pkg/apis/constraints"
 	"github.com/open-policy-agent/frameworks/constraint/pkg/types"
 	"github.com/open-policy-agent/gatekeeper/apis"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -86,7 +88,17 @@ func (r *Runner) skipTest(t Test) TestResult {
 func (r *Runner) runTest(ctx context.Context, suiteDir string, filter Filter, t Test) TestResult {
 	start := time.Now()
 
-	results, err := r.runCases(ctx, suiteDir, filter, t)
+	err := r.tryAddConstraint(ctx, suiteDir, t)
+	var results []CaseResult
+	if t.Invalid {
+		if errors.Is(err, constraints.ErrSchema) {
+			err = nil
+		} else {
+			err = fmt.Errorf("%w: got error %v but want %v", ErrValidConstraint, err, constraints.ErrSchema)
+		}
+	} else if err == nil {
+		results, err = r.runCases(ctx, suiteDir, filter, t)
+	}
 
 	return TestResult{
 		Name:        t.Name,
@@ -96,6 +108,31 @@ func (r *Runner) runTest(ctx context.Context, suiteDir string, filter Filter, t
 	}
 }
 
+func (r *Runner) tryAddConstraint(ctx context.Context, suiteDir string, t Test) error {
+	client, err := r.newClient()
+	if err != nil {
+		return fmt.Errorf("%w: %v", ErrCreatingClient, err)
+	}
+
+	err = r.addTemplate(suiteDir, t.Template, client)
+	if err != nil {
+		return err
+	}
+
+	constraintPath := t.Constraint
+	if constraintPath == "" {
+		return fmt.Errorf("%w: missing constraint", ErrInvalidSuite)
+	}
+
+	cObj, err := readConstraint(r.filesystem, path.Join(suiteDir, constraintPath))
+	if err != nil {
+		return err
+	}
+
+	_, err = client.AddConstraint(ctx, cObj)
+	return err
+}
+
 // runCases executes every Case in the Test. Returns the results for every Case,
 // or an error if there was a problem executing the Test.
 func (r *Runner) runCases(ctx context.Context, suiteDir string, filter Filter, t Test) ([]CaseResult, error) {
@@ -108,11 +145,6 @@ func (r *Runner) runCases(ctx context.Context, suiteDir string, filter Filter, t
 		return c, nil
 	}
 
-	_, err := newClient()
-	if err != nil {
-		return nil, err
-	}
-
 	results := make([]CaseResult, len(t.Cases))
 
 	for i, c := range t.Cases {

pkg/gator/runner_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client"
 	"github.com/open-policy-agent/gatekeeper/pkg/gator/fixtures"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
@@ -158,6 +159,50 @@ func TestRunner_Run(t *testing.T) {
 				}},
 			},
 		},
+		{
+			name: "want invalid Constraint",
+			suite: Suite{
+				Tests: []Test{{
+					Template:   "allow-template.yaml",
+					Constraint: "allow-constraint.yaml",
+					Invalid:    true,
+				}},
+			},
+			f: fstest.MapFS{
+				"allow-template.yaml": &fstest.MapFile{
+					Data: []byte(fixtures.TemplateRequiredLabel),
+				},
+				"allow-constraint.yaml": &fstest.MapFile{
+					Data: []byte(fixtures.ConstraintRequireLabelInvalid),
+				},
+			},
+			want: SuiteResult{
+				TestResults: []TestResult{{}},
+			},
+		},
+		{
+			name: "want invalid Constraint but Constraint valid",
+			suite: Suite{
+				Tests: []Test{{
+					Template:   "allow-template.yaml",
+					Constraint: "allow-constraint.yaml",
+					Invalid:    true,
+				}},
+			},
+			f: fstest.MapFS{
+				"allow-template.yaml": &fstest.MapFile{
+					Data: []byte(fixtures.TemplateRequiredLabel),
+				},
+				"allow-constraint.yaml": &fstest.MapFile{
+					Data: []byte(fixtures.ConstraintRequireLabelValid),
+				},
+			},
+			want: SuiteResult{
+				TestResults: []TestResult{{
+					Error: ErrValidConstraint,
+				}},
+			},
+		},
 		{
 			name: "valid Suite",
 			suite: Suite{
@@ -599,7 +644,7 @@ func TestRunner_Run(t *testing.T) {
 			},
 			want: SuiteResult{
 				TestResults: []TestResult{{
-					Error: ErrAddingConstraint,
+					Error: constraintclient.ErrMissingConstraintTemplate,
 				}},
 			},
 		},

pkg/gator/suite.go

@@ -33,8 +33,15 @@ type Test struct {
 	Constraint string `json:"constraint"`
 
 	// Cases are the test cases to run on the instantiated Constraint.
+	// Mutually exclusive with Invalid.
 	Cases []*Case `json:"cases,omitempty"`
 
+	// Invalid, if true, specifies that the Constraint is expected to be invalid
+	// for the Template. For example - a required field is missing or is of the
+	// wrong type.
+	// Mutually exclusive with Cases.
+	Invalid bool `json:"invalid"`
+
 	// Skip, if true, skips this Test.
 	Skip bool `json:"skip"`
 }

vendor/modules.txt

@@ -295,7 +295,7 @@ github.com/onsi/gomega/types
 # github.com/open-policy-agent/cert-controller v0.3.0
 ## explicit; go 1.17
 github.com/open-policy-agent/cert-controller/pkg/rotator
-# github.com/open-policy-agent/frameworks/constraint v0.0.0-20220527154834-b482a682709a
+# github.com/open-policy-agent/frameworks/constraint v0.0.0-20220603201627-e8d94b2fcf7a
 ## explicit; go 1.17
 github.com/open-policy-agent/frameworks/constraint/deploy
 github.com/open-policy-agent/frameworks/constraint/pkg/apis