feat: add syncset controller, feat: syncset readiness (#3030)

Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com>
Signed-off-by: alex <8968914+acpana@users.noreply.github.com>
Co-authored-by: Rita Zhang <rita.z.zhang@gmail.com>

apis/config/v1alpha1/config_types.go

@@ -18,6 +18,7 @@ package v1alpha1
 import (
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
 // ConfigSpec defines the desired state of Config.
@@ -62,6 +63,14 @@ type SyncOnlyEntry struct {
 	Kind    string `json:"kind,omitempty"`
 }
 
+func (e *SyncOnlyEntry) ToGroupVersionKind() schema.GroupVersionKind {
+	return schema.GroupVersionKind{
+		Group:   e.Group,
+		Version: e.Version,
+		Kind:    e.Kind,
+	}
+}
+
 type MatchEntry struct {
 	Processes          []string            `json:"processes,omitempty"`
 	ExcludedNamespaces []wildcard.Wildcard `json:"excludedNamespaces,omitempty"`

apis/syncset/v1alpha1/syncset_types.go

@@ -2,6 +2,7 @@ package v1alpha1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
 type SyncSetSpec struct {
@@ -14,10 +15,18 @@ type GVKEntry struct {
 	Kind    string `json:"kind,omitempty"`
 }
 
+func (e *GVKEntry) ToGroupVersionKind() schema.GroupVersionKind {
+	return schema.GroupVersionKind{
+		Group:   e.Group,
+		Version: e.Version,
+		Kind:    e.Kind,
+	}
+}
+
 // +kubebuilder:resource:scope=Cluster
 // +kubebuilder:object:root=true
 
-// SyncSet is the Schema for the SyncSet API.
+// SyncSet defines which resources Gatekeeper will cache. The union of all SyncSets plus the syncOnly field of Gatekeeper's Config resource defines the sets of resources that will be synced.
 type SyncSet struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

config/crd/bases/syncset.gatekeeper.sh_syncsets.yaml

@@ -18,7 +18,9 @@ spec:
   - name: v1alpha1
     schema:
       openAPIV3Schema:
-        description: SyncSet is the Schema for the SyncSet API.
+        description: SyncSet defines which resources Gatekeeper will cache. The union
+          of all SyncSets plus the syncOnly field of Gatekeeper's Config resource
+          defines the sets of resources that will be synced.
         properties:
           apiVersion:
             description: 'APIVersion defines the versioned schema of this representation

config/crd/kustomization.yaml

@@ -3,7 +3,7 @@
 # It should be run by config/default
 resources:
 - bases/config.gatekeeper.sh_configs.yaml
-#- bases/syncset.gatekeeper.sh_syncsets.yaml
+- bases/syncset.gatekeeper.sh_syncsets.yaml
 - bases/status.gatekeeper.sh_constraintpodstatuses.yaml
 - bases/status.gatekeeper.sh_constrainttemplatepodstatuses.yaml
 - bases/status.gatekeeper.sh_mutatorpodstatuses.yaml
@@ -56,6 +56,12 @@ patchesJson6902:
     kind: CustomResourceDefinition
     name: expansiontemplate.expansion.gatekeeper.sh
   path: patches/max_name_size.yaml
+- target:
+    group: apiextensions.k8s.io
+    version: v1
+    kind: CustomResourceDefinition
+    name: syncsets.syncset.gatekeeper.sh
+  path: patches/max_name_size.yaml
 
 patchesStrategicMerge:
 #- patches/max_name_size_for_modifyset.yaml

main.go

@@ -53,6 +53,7 @@ import (
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/operations"
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub"
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/readiness"
+	"github.com/open-policy-agent/gatekeeper/v3/pkg/readiness/pruner"
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil"
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/target"
 	"github.com/open-policy-agent/gatekeeper/v3/pkg/upgrade"
@@ -486,6 +487,12 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle
 		return err
 	}
 
+	err = mgr.Add(pruner.NewExpectationsPruner(cm, tracker))
+	if err != nil {
+		setupLog.Error(err, "adding expectations pruner to manager")
+		return err
+	}
+
 	opts := controller.Dependencies{
 		CFClient:         client,
 		WatchManger:      wm,

manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml

@@ -0,0 +1,52 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.10.0
+  labels:
+    gatekeeper.sh/system: "yes"
+  name: syncsets.syncset.gatekeeper.sh
+spec:
+  group: syncset.gatekeeper.sh
+  names:
+    kind: SyncSet
+    listKind: SyncSetList
+    plural: syncsets
+    singular: syncset
+  preserveUnknownFields: false
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: SyncSet defines which resources Gatekeeper will cache. The union of all SyncSets plus the syncOnly field of Gatekeeper's Config resource defines the sets of resources that will be synced.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            properties:
+              name:
+                maxLength: 63
+                type: string
+            type: object
+          spec:
+            properties:
+              gvks:
+                items:
+                  properties:
+                    group:
+                      type: string
+                    kind:
+                      type: string
+                    version:
+                      type: string
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true

manifest_staging/deploy/gatekeeper.yaml

@@ -3371,6 +3371,59 @@ spec:
     served: true
     storage: true
 ---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.10.0
+  labels:
+    gatekeeper.sh/system: "yes"
+  name: syncsets.syncset.gatekeeper.sh
+spec:
+  group: syncset.gatekeeper.sh
+  names:
+    kind: SyncSet
+    listKind: SyncSetList
+    plural: syncsets
+    singular: syncset
+  preserveUnknownFields: false
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: SyncSet defines which resources Gatekeeper will cache. The union of all SyncSets plus the syncOnly field of Gatekeeper's Config resource defines the sets of resources that will be synced.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            properties:
+              name:
+                maxLength: 63
+                type: string
+            type: object
+          spec:
+            properties:
+              gvks:
+                items:
+                  properties:
+                    group:
+                      type: string
+                    kind:
+                      type: string
+                    version:
+                      type: string
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:

pkg/cachemanager/aggregator/aggregator.go

@@ -1,7 +1,6 @@
 package aggregator
 
 import (
-	"fmt"
 	gosync "sync"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -10,9 +9,9 @@ import (
 // Key defines a type, identifier tuple to store
 // in the GVKAggregator.
 type Key struct {
-	// Source specifies the type or where this Key comes from.
+	// Source specifies the type of the source object.
 	Source string
-	// ID specifies the name of the type that this Key is.
+	// ID specifies the name of the instance of the source object.
 	ID string
 }
 
@@ -49,46 +48,39 @@ func (b *GVKAgreggator) IsPresent(gvk schema.GroupVersionKind) bool {
 // Remove deletes any associations that Key k has in the GVKAggregator.
 // For any GVK in the association k --> [GVKs], we also delete any associations
 // between the GVK and the Key k stored in the reverse map.
-func (b *GVKAgreggator) Remove(k Key) error {
+func (b *GVKAgreggator) Remove(k Key) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
 	gvks, found := b.store[k]
 	if !found {
-		return nil
+		return
 	}
 
-	if err := b.pruneReverseStore(gvks, k); err != nil {
-		return err
-	}
+	b.pruneReverseStore(gvks, k)
 
 	delete(b.store, k)
-	return nil
 }
 
 // Upsert stores an association between Key k and the list of GVKs
-// and also the reverse associatoin between each GVK passed in and Key k.
+// and also the reverse association between each GVK passed in and Key k.
 // Any old associations are dropped, unless they are included in the new list of
 // GVKs.
-// It errors out if there is an internal issue with remove the reverse Key links
-// for any GVKs that are being dropped as part of this Upsert call.
-func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) error {
+func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
 	oldGVKs, found := b.store[k]
 	if found {
 		// gvksToRemove contains old GKVs that are not included in the new gvks list
 		gvksToRemove := unreferencedOldGVKsToPrune(gvks, oldGVKs)
-		if err := b.pruneReverseStore(gvksToRemove, k); err != nil {
-			return fmt.Errorf("failed to prune entries on upsert: %w", err)
-		}
+		b.pruneReverseStore(gvksToRemove, k)
 	}
 
 	// protect against empty inputs
 	gvksSet := makeSet(gvks)
 	if len(gvksSet) == 0 {
-		return nil
+		return
 	}
 
 	b.store[k] = gvksSet
@@ -99,19 +91,17 @@ func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) error {
 		}
 		b.reverseStore[gvk][k] = struct{}{}
 	}
-
-	return nil
 }
 
 // List returnes the gvk set for a given Key.
-func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} {
+func (b *GVKAgreggator) List(k Key) []schema.GroupVersionKind {
 	b.mu.RLock()
 	defer b.mu.RUnlock()
 
 	v := b.store[k]
-	cpy := make(map[schema.GroupVersionKind]struct{}, len(v))
-	for key, value := range v {
-		cpy[key] = value
+	cpy := []schema.GroupVersionKind{}
+	for key := range v {
+		cpy = append(cpy, key)
 	}
 	return cpy
 }
@@ -128,13 +118,12 @@ func (b *GVKAgreggator) GVKs() []schema.GroupVersionKind {
 	return allGVKs
 }
 
-func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struct{}, k Key) error {
+func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struct{}, k Key) {
 	for gvk := range gvks {
 		keySet, found := b.reverseStore[gvk]
-		if !found || len(keySet) == 0 {
-			// this should not happen if we keep the two maps well defined
-			// but let's be defensive nonetheless.
-			return fmt.Errorf("internal aggregator error: gvks stores are corrupted for key: %s", k)
+		if !found {
+			// by definition, nothing to prune
+			return
 		}
 
 		delete(keySet, k)
@@ -146,8 +135,6 @@ func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struc
 			b.reverseStore[gvk] = keySet
 		}
 	}
-
-	return nil
 }
 
 func makeSet(gvks []schema.GroupVersionKind) map[schema.GroupVersionKind]struct{} {