diff --git a/apis/expansion/unversioned/expansiontemplate_types.go b/apis/expansion/unversioned/expansiontemplate_types.go index 74d3385a895..91bab506b30 100644 --- a/apis/expansion/unversioned/expansiontemplate_types.go +++ b/apis/expansion/unversioned/expansiontemplate_types.go @@ -16,6 +16,7 @@ limitations under the License. package unversioned import ( + statusv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" "github.com/open-policy-agent/gatekeeper/pkg/mutation/match" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -63,7 +64,13 @@ type ExpansionTemplate struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec ExpansionTemplateSpec `json:"spec,omitempty"` + Spec ExpansionTemplateSpec `json:"spec,omitempty"` + Status ExpansionTemplateStatus `json:"status,omitempty"` +} + +// ExpansionTemplateStatus defines the observed state of ExpansionTemplate. +type ExpansionTemplateStatus struct { + ByPod []statusv1alpha1.ExpansionTemplatePodStatusStatus `json:"byPod,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/expansion/unversioned/zz_generated.deepcopy.go b/apis/expansion/unversioned/zz_generated.deepcopy.go index c3b73060f42..360bbe7849b 100644 --- a/apis/expansion/unversioned/zz_generated.deepcopy.go +++ b/apis/expansion/unversioned/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package unversioned import ( + "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" "github.com/open-policy-agent/gatekeeper/pkg/mutation/match" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -31,6 +32,7 @@ func (in *ExpansionTemplate) DeepCopyInto(out *ExpansionTemplate) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplate. @@ -106,6 +108,28 @@ func (in *ExpansionTemplateSpec) DeepCopy() *ExpansionTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpansionTemplateStatus) DeepCopyInto(out *ExpansionTemplateStatus) { + *out = *in + if in.ByPod != nil { + in, out := &in.ByPod, &out.ByPod + *out = make([]v1beta1.ExpansionTemplatePodStatusStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplateStatus. +func (in *ExpansionTemplateStatus) DeepCopy() *ExpansionTemplateStatus { + if in == nil { + return nil + } + out := new(ExpansionTemplateStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GeneratedGVK) DeepCopyInto(out *GeneratedGVK) { *out = *in diff --git a/apis/expansion/v1alpha1/expansion_template_types.go b/apis/expansion/v1alpha1/expansiontemplate_types.go similarity index 86% rename from apis/expansion/v1alpha1/expansion_template_types.go rename to apis/expansion/v1alpha1/expansiontemplate_types.go index 3b710013f56..6dcb52938f0 100644 --- a/apis/expansion/v1alpha1/expansion_template_types.go +++ b/apis/expansion/v1alpha1/expansiontemplate_types.go @@ -16,6 +16,7 @@ limitations under the License. package v1alpha1 import ( + status "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" "github.com/open-policy-agent/gatekeeper/pkg/mutation/match" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -56,13 +57,21 @@ type GeneratedGVK struct { // +kubebuilder:object:root=true // +kubebuilder:resource:path="expansiontemplate" // +kubebuilder:resource:scope="Cluster" +// +kubebuilder:subresource:status +// +kubebuilder:storageversion // ExpansionTemplate is the Schema for the ExpansionTemplate API. type ExpansionTemplate struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec ExpansionTemplateSpec `json:"spec,omitempty"` + Spec ExpansionTemplateSpec `json:"spec,omitempty"` + Status ExpansionTemplateStatus `json:"status,omitempty"` +} + +// ExpansionTemplateStatus defines the observed state of ExpansionTemplate. +type ExpansionTemplateStatus struct { + ByPod []status.ExpansionTemplatePodStatusStatus `json:"byPod,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/expansion/v1alpha1/zz_generated.conversion.go b/apis/expansion/v1alpha1/zz_generated.conversion.go index 16c04fddadb..43871d34ad0 100644 --- a/apis/expansion/v1alpha1/zz_generated.conversion.go +++ b/apis/expansion/v1alpha1/zz_generated.conversion.go @@ -23,6 +23,7 @@ import ( unsafe "unsafe" unversioned "github.com/open-policy-agent/gatekeeper/apis/expansion/unversioned" + v1beta1 "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" match "github.com/open-policy-agent/gatekeeper/pkg/mutation/match" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" @@ -65,6 +66,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ExpansionTemplateStatus)(nil), (*unversioned.ExpansionTemplateStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ExpansionTemplateStatus_To_unversioned_ExpansionTemplateStatus(a.(*ExpansionTemplateStatus), b.(*unversioned.ExpansionTemplateStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*unversioned.ExpansionTemplateStatus)(nil), (*ExpansionTemplateStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_unversioned_ExpansionTemplateStatus_To_v1alpha1_ExpansionTemplateStatus(a.(*unversioned.ExpansionTemplateStatus), b.(*ExpansionTemplateStatus), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*GeneratedGVK)(nil), (*unversioned.GeneratedGVK)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_GeneratedGVK_To_unversioned_GeneratedGVK(a.(*GeneratedGVK), b.(*unversioned.GeneratedGVK), scope) }); err != nil { @@ -83,6 +94,9 @@ func autoConvert_v1alpha1_ExpansionTemplate_To_unversioned_ExpansionTemplate(in if err := Convert_v1alpha1_ExpansionTemplateSpec_To_unversioned_ExpansionTemplateSpec(&in.Spec, &out.Spec, s); err != nil { return err } + if err := Convert_v1alpha1_ExpansionTemplateStatus_To_unversioned_ExpansionTemplateStatus(&in.Status, &out.Status, s); err != nil { + return err + } return nil } @@ -96,6 +110,9 @@ func autoConvert_unversioned_ExpansionTemplate_To_v1alpha1_ExpansionTemplate(in if err := Convert_unversioned_ExpansionTemplateSpec_To_v1alpha1_ExpansionTemplateSpec(&in.Spec, &out.Spec, s); err != nil { return err } + if err := Convert_unversioned_ExpansionTemplateStatus_To_v1alpha1_ExpansionTemplateStatus(&in.Status, &out.Status, s); err != nil { + return err + } return nil } @@ -156,6 +173,26 @@ func Convert_unversioned_ExpansionTemplateSpec_To_v1alpha1_ExpansionTemplateSpec return autoConvert_unversioned_ExpansionTemplateSpec_To_v1alpha1_ExpansionTemplateSpec(in, out, s) } +func autoConvert_v1alpha1_ExpansionTemplateStatus_To_unversioned_ExpansionTemplateStatus(in *ExpansionTemplateStatus, out *unversioned.ExpansionTemplateStatus, s conversion.Scope) error { + out.ByPod = *(*[]v1beta1.ExpansionTemplatePodStatusStatus)(unsafe.Pointer(&in.ByPod)) + return nil +} + +// Convert_v1alpha1_ExpansionTemplateStatus_To_unversioned_ExpansionTemplateStatus is an autogenerated conversion function. +func Convert_v1alpha1_ExpansionTemplateStatus_To_unversioned_ExpansionTemplateStatus(in *ExpansionTemplateStatus, out *unversioned.ExpansionTemplateStatus, s conversion.Scope) error { + return autoConvert_v1alpha1_ExpansionTemplateStatus_To_unversioned_ExpansionTemplateStatus(in, out, s) +} + +func autoConvert_unversioned_ExpansionTemplateStatus_To_v1alpha1_ExpansionTemplateStatus(in *unversioned.ExpansionTemplateStatus, out *ExpansionTemplateStatus, s conversion.Scope) error { + out.ByPod = *(*[]v1beta1.ExpansionTemplatePodStatusStatus)(unsafe.Pointer(&in.ByPod)) + return nil +} + +// Convert_unversioned_ExpansionTemplateStatus_To_v1alpha1_ExpansionTemplateStatus is an autogenerated conversion function. +func Convert_unversioned_ExpansionTemplateStatus_To_v1alpha1_ExpansionTemplateStatus(in *unversioned.ExpansionTemplateStatus, out *ExpansionTemplateStatus, s conversion.Scope) error { + return autoConvert_unversioned_ExpansionTemplateStatus_To_v1alpha1_ExpansionTemplateStatus(in, out, s) +} + func autoConvert_v1alpha1_GeneratedGVK_To_unversioned_GeneratedGVK(in *GeneratedGVK, out *unversioned.GeneratedGVK, s conversion.Scope) error { out.Group = in.Group out.Version = in.Version diff --git a/apis/expansion/v1alpha1/zz_generated.deepcopy.go b/apis/expansion/v1alpha1/zz_generated.deepcopy.go index 8b08913b818..0c35193dba1 100644 --- a/apis/expansion/v1alpha1/zz_generated.deepcopy.go +++ b/apis/expansion/v1alpha1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1alpha1 import ( + "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" "github.com/open-policy-agent/gatekeeper/pkg/mutation/match" "k8s.io/apimachinery/pkg/runtime" ) @@ -31,6 +32,7 @@ func (in *ExpansionTemplate) DeepCopyInto(out *ExpansionTemplate) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplate. @@ -106,6 +108,28 @@ func (in *ExpansionTemplateSpec) DeepCopy() *ExpansionTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpansionTemplateStatus) DeepCopyInto(out *ExpansionTemplateStatus) { + *out = *in + if in.ByPod != nil { + in, out := &in.ByPod, &out.ByPod + *out = make([]v1beta1.ExpansionTemplatePodStatusStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplateStatus. +func (in *ExpansionTemplateStatus) DeepCopy() *ExpansionTemplateStatus { + if in == nil { + return nil + } + out := new(ExpansionTemplateStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GeneratedGVK) DeepCopyInto(out *GeneratedGVK) { *out = *in diff --git a/apis/status/v1beta1/constraintpodstatus_types.go b/apis/status/v1beta1/constraintpodstatus_types.go index e91fca1dfeb..e344b7d8063 100644 --- a/apis/status/v1beta1/constraintpodstatus_types.go +++ b/apis/status/v1beta1/constraintpodstatus_types.go @@ -113,5 +113,5 @@ func KeyForConstraint(id string, constraint *unstructured.Unstructured) (string, // because K8s requires all lowercase letters for resource names kind := strings.ToLower(constraint.GetObjectKind().GroupVersionKind().Kind) name := constraint.GetName() - return dashPacker(id, kind, name) + return DashPacker(id, kind, name) } diff --git a/apis/status/v1beta1/constrainttemplatepodstatus_types.go b/apis/status/v1beta1/constrainttemplatepodstatus_types.go index 1b9e0e77399..356d53437f3 100644 --- a/apis/status/v1beta1/constrainttemplatepodstatus_types.go +++ b/apis/status/v1beta1/constrainttemplatepodstatus_types.go @@ -88,5 +88,5 @@ func NewConstraintTemplateStatusForPod(pod *corev1.Pod, templateName string, sch // KeyForConstraintTemplate returns a unique status object name given the Pod ID and // a template object. func KeyForConstraintTemplate(id string, templateName string) (string, error) { - return dashPacker(id, templateName) + return DashPacker(id, templateName) } diff --git a/apis/status/v1beta1/expansiontemplatepodstatus_types.go b/apis/status/v1beta1/expansiontemplatepodstatus_types.go new file mode 100644 index 00000000000..067da872862 --- /dev/null +++ b/apis/status/v1beta1/expansiontemplatepodstatus_types.go @@ -0,0 +1,83 @@ +package v1beta1 + +import ( + "github.com/open-policy-agent/gatekeeper/pkg/operations" + "github.com/open-policy-agent/gatekeeper/pkg/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. +type ExpansionTemplatePodStatusStatus struct { + // Important: Run "make" to regenerate code after modifying this file + ID string `json:"id,omitempty"` + TemplateUID types.UID `json:"templateUID,omitempty"` + Operations []string `json:"operations,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Errors []*ExpansionTemplateError `json:"errors,omitempty"` +} + +// +kubebuilder:object:generate=true + +type ExpansionTemplateError struct { + Type string `json:"type,omitempty"` + Message string `json:"message"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced + +// ExpansionTemplatePodStatus is the Schema for the expansiontemplatepodstatuses API. +type ExpansionTemplatePodStatus struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Status ExpansionTemplatePodStatusStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ExpansionTemplatePodStatusList contains a list of ExpansionTemplatePodStatus. +type ExpansionTemplatePodStatusList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ExpansionTemplatePodStatus `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ExpansionTemplatePodStatus{}, &ExpansionTemplatePodStatusList{}) +} + +// NewExpansionTemplateStatusForPod returns an expansion template status object +// that has been initialized with the bare minimum of fields to make it functional +// with the expansion template status controller. +func NewExpansionTemplateStatusForPod(pod *corev1.Pod, templateName string, scheme *runtime.Scheme) (*ExpansionTemplatePodStatus, error) { + obj := &ExpansionTemplatePodStatus{} + name, err := KeyForExpansionTemplate(pod.Name, templateName) + if err != nil { + return nil, err + } + obj.SetName(name) + obj.SetNamespace(util.GetNamespace()) + obj.Status.ID = pod.Name + obj.Status.Operations = operations.AssignedStringList() + obj.SetLabels(map[string]string{ + ExpansionTemplateNameLabel: templateName, + PodLabel: pod.Name, + }) + + if err := controllerutil.SetOwnerReference(pod, obj, scheme); err != nil { + return nil, err + } + + return obj, nil +} + +// KeyForExpansionTemplate returns a unique status object name given the Pod ID and +// a template object. +func KeyForExpansionTemplate(id string, templateName string) (string, error) { + return DashPacker(id, templateName) +} diff --git a/apis/status/v1beta1/labels.go b/apis/status/v1beta1/labels.go index 9456c8f948f..0f0caca91ce 100644 --- a/apis/status/v1beta1/labels.go +++ b/apis/status/v1beta1/labels.go @@ -2,6 +2,7 @@ package v1beta1 // Label keys used for internal gatekeeper operations. const ( + ExpansionTemplateNameLabel = "internal.gatekeeper.sh/expansiontemplate-name" ConstraintNameLabel = "internal.gatekeeper.sh/constraint-name" ConstraintKindLabel = "internal.gatekeeper.sh/constraint-kind" ConstraintTemplateNameLabel = "internal.gatekeeper.sh/constrainttemplate-name" diff --git a/apis/status/v1beta1/mutatorpodstatus_types.go b/apis/status/v1beta1/mutatorpodstatus_types.go index 83ed175d931..58e76faaa0b 100644 --- a/apis/status/v1beta1/mutatorpodstatus_types.go +++ b/apis/status/v1beta1/mutatorpodstatus_types.go @@ -113,5 +113,5 @@ func KeyForMutatorID(id string, mID mtypes.ID) (string, error) { // We must do this because K8s requires all lowercase letters for resource names kind := strings.ToLower(mID.Kind) name := mID.Name - return dashPacker(id, kind, name) + return DashPacker(id, kind, name) } diff --git a/apis/status/v1beta1/util.go b/apis/status/v1beta1/util.go index 58f1cbd117b..decb0905526 100644 --- a/apis/status/v1beta1/util.go +++ b/apis/status/v1beta1/util.go @@ -32,7 +32,7 @@ func dashExtractor(val string) []string { return tokens } -// dashPacker puts a list of strings into a dash-separated format. Note that +// DashPacker puts a list of strings into a dash-separated format. Note that // it cannot handle empty strings, as that makes the dash separator for the empty // string reduce to an escaped dash. This is fine because none of the packed strings // are allowed to be empty. If this changes in the future, we could create a placeholder @@ -43,17 +43,17 @@ func dashExtractor(val string) []string { // which is also disallowed by the schema (and would require an additional placeholder // character to fix). Finally, note that it is impossible to distinguish between // a nil list of strings and a list of one empty string. -func dashPacker(vals ...string) (string, error) { +func DashPacker(vals ...string) (string, error) { if len(vals) == 0 { - return "", fmt.Errorf("dashPacker cannot pack an empty list of strings") + return "", fmt.Errorf("DashPacker cannot pack an empty list of strings") } b := strings.Builder{} for i, val := range vals { if strings.HasPrefix(val, "-") || strings.HasSuffix(val, "-") { - return "", fmt.Errorf("dashPacker cannot pack strings that begin or end with a dash: %+v", vals) + return "", fmt.Errorf("DashPacker cannot pack strings that begin or end with a dash: %+v", vals) } if len(val) == 0 { - return "", fmt.Errorf("dashPacker cannot pack empty strings: %v", vals) + return "", fmt.Errorf("DashPacker cannot pack empty strings: %v", vals) } if i != 0 { b.WriteString("-") diff --git a/apis/status/v1beta1/util_test.go b/apis/status/v1beta1/util_test.go index 592b9bd1226..47e4a58db20 100644 --- a/apis/status/v1beta1/util_test.go +++ b/apis/status/v1beta1/util_test.go @@ -91,12 +91,12 @@ var dashingTestCases = []struct { func TestDashPacker(t *testing.T) { for _, tc := range dashingTestCases { t.Run(tc.name, func(t *testing.T) { - gotPacked, err := dashPacker(tc.extracted...) + gotPacked, err := DashPacker(tc.extracted...) if err != nil { t.Fatal(err) } if diff := cmp.Diff(tc.packed, gotPacked); diff != "" { - t.Fatal("got dashPacker(tc.extracted...) != tc.packed, want equal") + t.Fatal("got DashPacker(tc.extracted...) != tc.packed, want equal") } }) } diff --git a/apis/status/v1beta1/zz_generated.deepcopy.go b/apis/status/v1beta1/zz_generated.deepcopy.go index dddc9496679..5a42224d349 100644 --- a/apis/status/v1beta1/zz_generated.deepcopy.go +++ b/apis/status/v1beta1/zz_generated.deepcopy.go @@ -212,6 +212,110 @@ func (in *Error) DeepCopy() *Error { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpansionTemplateError) DeepCopyInto(out *ExpansionTemplateError) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplateError. +func (in *ExpansionTemplateError) DeepCopy() *ExpansionTemplateError { + if in == nil { + return nil + } + out := new(ExpansionTemplateError) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpansionTemplatePodStatus) DeepCopyInto(out *ExpansionTemplatePodStatus) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplatePodStatus. +func (in *ExpansionTemplatePodStatus) DeepCopy() *ExpansionTemplatePodStatus { + if in == nil { + return nil + } + out := new(ExpansionTemplatePodStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ExpansionTemplatePodStatus) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpansionTemplatePodStatusList) DeepCopyInto(out *ExpansionTemplatePodStatusList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ExpansionTemplatePodStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplatePodStatusList. +func (in *ExpansionTemplatePodStatusList) DeepCopy() *ExpansionTemplatePodStatusList { + if in == nil { + return nil + } + out := new(ExpansionTemplatePodStatusList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ExpansionTemplatePodStatusList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpansionTemplatePodStatusStatus) DeepCopyInto(out *ExpansionTemplatePodStatusStatus) { + *out = *in + if in.Operations != nil { + in, out := &in.Operations, &out.Operations + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Errors != nil { + in, out := &in.Errors, &out.Errors + *out = make([]*ExpansionTemplateError, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ExpansionTemplateError) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpansionTemplatePodStatusStatus. +func (in *ExpansionTemplatePodStatusStatus) DeepCopy() *ExpansionTemplatePodStatusStatus { + if in == nil { + return nil + } + out := new(ExpansionTemplatePodStatusStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MutatorError) DeepCopyInto(out *MutatorError) { *out = *in diff --git a/cmd/build/helmify/kustomization.yaml b/cmd/build/helmify/kustomization.yaml index e6507f7114d..a7c4a4647ac 100644 --- a/cmd/build/helmify/kustomization.yaml +++ b/cmd/build/helmify/kustomization.yaml @@ -34,6 +34,12 @@ patchesJson6902: kind: CustomResourceDefinition name: mutatorpodstatuses.status.gatekeeper.sh path: labels_patch.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: expansiontemplatepodstatuses.status.gatekeeper.sh + path: labels_patch.yaml - target: group: apiextensions.k8s.io version: v1 diff --git a/config/crd/bases/expansion.gatekeeper.sh_expansiontemplate.yaml b/config/crd/bases/expansion.gatekeeper.sh_expansiontemplate.yaml index 12f7f40d95e..57570c632c4 100644 --- a/config/crd/bases/expansion.gatekeeper.sh_expansiontemplate.yaml +++ b/config/crd/bases/expansion.gatekeeper.sh_expansiontemplate.yaml @@ -79,6 +79,47 @@ spec: generators, this is usually spec.template type: string type: object + status: + description: ExpansionTemplateStatus defines the observed state of ExpansionTemplate. + properties: + byPod: + items: + description: ExpansionTemplatePodStatusStatus defines the observed + state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after + modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including + UUIDs. Because we don't ONLY use UUIDs, this is an alias + to string. Being a type captures intent and helps make sure + that UIDs and names do not get conflated. + type: string + type: object + type: array + type: object type: object served: true storage: true + subresources: + status: {} diff --git a/config/crd/bases/status.gatekeeper.sh_expansiontemplatepodstatuses.yaml b/config/crd/bases/status.gatekeeper.sh_expansiontemplatepodstatuses.yaml new file mode 100644 index 00000000000..4335d45f5ca --- /dev/null +++ b/config/crd/bases/status.gatekeeper.sh_expansiontemplatepodstatuses.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: expansiontemplatepodstatuses.status.gatekeeper.sh +spec: + group: status.gatekeeper.sh + names: + kind: ExpansionTemplatePodStatus + listKind: ExpansionTemplatePodStatusList + plural: expansiontemplatepodstatuses + singular: expansiontemplatepodstatus + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ExpansionTemplatePodStatus is the Schema for the expansiontemplatepodstatuses + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: ExpansionTemplatePodStatusStatus defines the observed state + of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying + this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including + UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being + a type captures intent and helps make sure that UIDs and names do + not get conflated. + type: string + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index c2b8e350482..475b34b63de 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -6,6 +6,7 @@ resources: - bases/status.gatekeeper.sh_constraintpodstatuses.yaml - bases/status.gatekeeper.sh_constrainttemplatepodstatuses.yaml - bases/status.gatekeeper.sh_mutatorpodstatuses.yaml +- bases/status.gatekeeper.sh_expansiontemplatepodstatuses.yaml - bases/mutations.gatekeeper.sh_assign.yaml - bases/mutations.gatekeeper.sh_assignimage.yaml - bases/mutations.gatekeeper.sh_assignmetadata.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b81a0171252..cb1aaf143ae 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -76,6 +76,18 @@ rules: - patch - update - watch +- apiGroups: + - expansion.gatekeeper.sh + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - externaldata.gatekeeper.sh resources: diff --git a/main.go b/main.go index a4be1697904..a08a1cb41e3 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ import ( frameworksexternaldata "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" api "github.com/open-policy-agent/gatekeeper/apis" configv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1" + expansionv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/expansion/v1alpha1" mutationsv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/mutations/v1alpha1" mutationsv1beta1 "github.com/open-policy-agent/gatekeeper/apis/mutations/v1beta1" statusv1beta1 "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" @@ -116,6 +117,7 @@ func init() { _ = statusv1beta1.AddToScheme(scheme) _ = mutationsv1alpha1.AddToScheme(scheme) _ = mutationsv1beta1.AddToScheme(scheme) + _ = expansionv1alpha1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme flag.Var(disabledBuiltins, "disable-opa-builtin", "disable opa built-in function, this flag can be declared more than once.") @@ -259,7 +261,7 @@ func innerMain() int { sw := watch.NewSwitch() // Setup tracker and register readiness probe. - tracker, err := readiness.SetupTracker(mgr, mutation.Enabled(), *externaldata.ExternalDataEnabled) + tracker, err := readiness.SetupTracker(mgr, mutation.Enabled(), *externaldata.ExternalDataEnabled, *expansion.ExpansionEnabled) if err != nil { setupLog.Error(err, "unable to register readiness tracker") return 1 diff --git a/manifest_staging/charts/gatekeeper/crds/expansiontemplate-customresourcedefinition.yaml b/manifest_staging/charts/gatekeeper/crds/expansiontemplate-customresourcedefinition.yaml index 042249cf102..beae74edbf9 100644 --- a/manifest_staging/charts/gatekeeper/crds/expansiontemplate-customresourcedefinition.yaml +++ b/manifest_staging/charts/gatekeeper/crds/expansiontemplate-customresourcedefinition.yaml @@ -68,6 +68,42 @@ spec: description: TemplateSource specifies the source field on the generator resource to use as the base for expanded resource. For Pod-creating generators, this is usually spec.template type: string type: object + status: + description: ExpansionTemplateStatus defines the observed state of ExpansionTemplate. + properties: + byPod: + items: + description: ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. + type: string + type: object + type: array + type: object type: object served: true storage: true + subresources: + status: {} diff --git a/manifest_staging/charts/gatekeeper/crds/expansiontemplatepodstatus-customresourcedefinition.yaml b/manifest_staging/charts/gatekeeper/crds/expansiontemplatepodstatus-customresourcedefinition.yaml new file mode 100644 index 00000000000..8f49b4c5f7f --- /dev/null +++ b/manifest_staging/charts/gatekeeper/crds/expansiontemplatepodstatus-customresourcedefinition.yaml @@ -0,0 +1,62 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + labels: + gatekeeper.sh/system: "yes" + name: expansiontemplatepodstatuses.status.gatekeeper.sh +spec: + group: status.gatekeeper.sh + names: + kind: ExpansionTemplatePodStatus + listKind: ExpansionTemplatePodStatusList + plural: expansiontemplatepodstatuses + singular: expansiontemplatepodstatus + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ExpansionTemplatePodStatus is the Schema for the expansiontemplatepodstatuses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. + type: string + type: object + type: object + served: true + storage: true diff --git a/manifest_staging/charts/gatekeeper/templates/gatekeeper-manager-role-clusterrole.yaml b/manifest_staging/charts/gatekeeper/templates/gatekeeper-manager-role-clusterrole.yaml index a57b2b80c88..3e55923360c 100644 --- a/manifest_staging/charts/gatekeeper/templates/gatekeeper-manager-role-clusterrole.yaml +++ b/manifest_staging/charts/gatekeeper/templates/gatekeeper-manager-role-clusterrole.yaml @@ -82,6 +82,18 @@ rules: - patch - update - watch +- apiGroups: + - expansion.gatekeeper.sh + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - externaldata.gatekeeper.sh resources: diff --git a/manifest_staging/deploy/gatekeeper.yaml b/manifest_staging/deploy/gatekeeper.yaml index 98bc929522d..e4d3a734e6b 100644 --- a/manifest_staging/deploy/gatekeeper.yaml +++ b/manifest_staging/deploy/gatekeeper.yaml @@ -2351,6 +2351,105 @@ spec: description: TemplateSource specifies the source field on the generator resource to use as the base for expanded resource. For Pod-creating generators, this is usually spec.template type: string type: object + status: + description: ExpansionTemplateStatus defines the observed state of ExpansionTemplate. + properties: + byPod: + items: + description: ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. + type: string + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + labels: + gatekeeper.sh/system: "yes" + name: expansiontemplatepodstatuses.status.gatekeeper.sh +spec: + group: status.gatekeeper.sh + names: + kind: ExpansionTemplatePodStatus + listKind: ExpansionTemplatePodStatusList + plural: expansiontemplatepodstatuses + singular: expansiontemplatepodstatus + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ExpansionTemplatePodStatus is the Schema for the expansiontemplatepodstatuses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. + type: string + type: object type: object served: true storage: true @@ -3293,6 +3392,18 @@ rules: - patch - update - watch +- apiGroups: + - expansion.gatekeeper.sh + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - externaldata.gatekeeper.sh resources: diff --git a/pkg/controller/add_expansionstatus.go b/pkg/controller/add_expansionstatus.go new file mode 100644 index 00000000000..bb97ca2556c --- /dev/null +++ b/pkg/controller/add_expansionstatus.go @@ -0,0 +1,24 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/open-policy-agent/gatekeeper/pkg/controller/expansionstatus" +) + +func init() { + Injectors = append(Injectors, &expansionstatus.Adder{}) +} diff --git a/pkg/controller/config/config_controller_suite_test.go b/pkg/controller/config/config_controller_suite_test.go index 82bd38f67b0..1ecb11af9c0 100644 --- a/pkg/controller/config/config_controller_suite_test.go +++ b/pkg/controller/config/config_controller_suite_test.go @@ -23,12 +23,9 @@ import ( "testing" "github.com/open-policy-agent/gatekeeper/apis" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/open-policy-agent/gatekeeper/test/testutils" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -52,7 +49,7 @@ func TestMain(m *testing.M) { stdlog.Fatal(err) } - if err := createGatekeeperNamespace(cfg); err != nil { + if err := testutils.CreateGatekeeperNamespace(cfg); err != nil { stdlog.Printf("creating namespace: %v", err) } @@ -74,22 +71,3 @@ func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan }) return fn, requests } - -// Bootstrap the gatekeeper-system namespace for use in tests. -func createGatekeeperNamespace(cfg *rest.Config) error { - c, err := client.New(cfg, client.Options{}) - if err != nil { - return err - } - - // Create gatekeeper namespace - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gatekeeper-system", - }, - } - - ctx := context.Background() - _, err = controllerutil.CreateOrUpdate(ctx, c, ns, func() error { return nil }) - return err -} diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 4bf600872a4..d843b201ca2 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -138,7 +138,7 @@ func TestReconcile(t *testing.T) { } cs := watch.NewSwitch() - tracker, err := readiness.SetupTracker(mgr, false, false) + tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { t.Fatal(err) } @@ -321,7 +321,7 @@ func TestConfig_DeleteSyncResources(t *testing.T) { } // set up tracker - tracker, err := readiness.SetupTracker(mgr, false, false) + tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { t.Fatal(err) } @@ -431,7 +431,7 @@ func TestConfig_CacheContents(t *testing.T) { opaClient := &fakeOpa{} cs := watch.NewSwitch() - tracker, err := readiness.SetupTracker(mgr, false, false) + tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { t.Fatal(err) } @@ -592,7 +592,7 @@ func TestConfig_Retries(t *testing.T) { opaClient := &fakeOpa{} cs := watch.NewSwitch() - tracker, err := readiness.SetupTracker(mgr, false, false) + tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { t.Fatal(err) } diff --git a/pkg/controller/constrainttemplate/constrainttemplate_controller_suite_test.go b/pkg/controller/constrainttemplate/constrainttemplate_controller_suite_test.go index b61197fd038..115c4b511cc 100644 --- a/pkg/controller/constrainttemplate/constrainttemplate_controller_suite_test.go +++ b/pkg/controller/constrainttemplate/constrainttemplate_controller_suite_test.go @@ -16,64 +16,14 @@ limitations under the License. package constrainttemplate import ( - "context" - "log" - "os" - "path/filepath" "testing" - "github.com/open-policy-agent/gatekeeper/apis" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/scheme" + "github.com/open-policy-agent/gatekeeper/test/testutils" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/envtest" ) var cfg *rest.Config func TestMain(m *testing.M) { - t := &envtest.Environment{ - CRDDirectoryPaths: []string{ - filepath.Join("..", "..", "..", "vendor", "github.com", "open-policy-agent", "frameworks", "constraint", "deploy", "crds.yaml"), - filepath.Join("..", "..", "..", "config", "crd", "bases"), - }, - ErrorIfCRDPathMissing: true, - } - if err := apis.AddToScheme(scheme.Scheme); err != nil { - log.Fatal(err) - } - - var err error - if cfg, err = t.Start(); err != nil { - log.Fatal(err) - } - log.Print("STARTED") - - code := m.Run() - if err = t.Stop(); err != nil { - log.Printf("error while trying to stop server: %v", err) - } - os.Exit(code) -} - -// Bootstrap the gatekeeper-system namespace for use in tests. -func createGatekeeperNamespace(cfg *rest.Config) error { - c, err := client.New(cfg, client.Options{}) - if err != nil { - return err - } - - // Create gatekeeper namespace - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gatekeeper-system", - }, - } - - ctx := context.Background() - _, err = controllerutil.CreateOrUpdate(ctx, c, ns, func() error { return nil }) - return err + testutils.StartControlPlane(m, &cfg, 3) } diff --git a/pkg/controller/constrainttemplate/constrainttemplate_controller_test.go b/pkg/controller/constrainttemplate/constrainttemplate_controller_test.go index 410923ab226..cfe2495c7ab 100644 --- a/pkg/controller/constrainttemplate/constrainttemplate_controller_test.go +++ b/pkg/controller/constrainttemplate/constrainttemplate_controller_test.go @@ -21,7 +21,6 @@ import ( "fmt" "strings" "testing" - "time" templatesv1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1" "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" @@ -35,70 +34,23 @@ import ( "github.com/open-policy-agent/gatekeeper/pkg/watch" testclient "github.com/open-policy-agent/gatekeeper/test/clients" "github.com/open-policy-agent/gatekeeper/test/testutils" - "github.com/open-policy-agent/gatekeeper/third_party/sigs.k8s.io/controller-runtime/pkg/dynamiccache" - "github.com/prometheus/client_golang/prometheus" "golang.org/x/net/context" admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics" ) -// constantRetry makes 3,000 attempts at a rate of 100 per second. Since this -// is a test instance and not a "real" cluster, this is fine and there's no need -// to increase the wait time each iteration. -var constantRetry = wait.Backoff{ - Steps: 3000, - Duration: 10 * time.Millisecond, -} - -// setupManager sets up a controller-runtime manager with registered watch manager. -func setupManager(t *testing.T) (manager.Manager, *watch.Manager) { - t.Helper() - - metrics.Registry = prometheus.NewRegistry() - mgr, err := manager.New(cfg, manager.Options{ - MetricsBindAddress: "0", - NewCache: dynamiccache.New, - MapperProvider: func(c *rest.Config) (meta.RESTMapper, error) { - return apiutil.NewDynamicRESTMapper(c) - }, - Logger: testutils.NewLogger(t), - }) - if err != nil { - t.Fatalf("setting up controller manager: %s", err) - } - c := mgr.GetCache() - dc, ok := c.(watch.RemovableCache) - if !ok { - t.Fatalf("expected dynamic cache, got: %T", c) - } - wm, err := watch.New(dc) - if err != nil { - t.Fatalf("could not create watch manager: %s", err) - } - if err := mgr.Add(wm); err != nil { - t.Fatalf("could not add watch manager to manager: %s", err) - } - return mgr, wm -} - func makeReconcileConstraintTemplate(suffix string) *v1beta1.ConstraintTemplate { return &v1beta1.ConstraintTemplate{ TypeMeta: metav1.TypeMeta{ @@ -151,12 +103,12 @@ func TestReconcile(t *testing.T) { // Setup the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a // channel when it is finished. - mgr, wm := setupManager(t) + mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) // creating the gatekeeper-system namespace is necessary because that's where // status resources live by default - err := createGatekeeperNamespace(mgr.GetConfig()) + err := testutils.CreateGatekeeperNamespace(mgr.GetConfig()) if err != nil { t.Fatal(err) } @@ -175,7 +127,7 @@ func TestReconcile(t *testing.T) { testutils.Setenv(t, "POD_NAME", "no-pod") cs := watch.NewSwitch() - tracker, err := readiness.SetupTracker(mgr, false, false) + tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { t.Fatal(err) } @@ -205,11 +157,11 @@ func TestReconcile(t *testing.T) { logger.Info("Running test: CRD Gets Created") constraintTemplate := makeReconcileConstraintTemplate(suffix) - t.Cleanup(deleteObjectAndConfirm(ctx, t, c, expectedCRD(suffix))) - createThenCleanup(ctx, t, c, constraintTemplate) + t.Cleanup(testutils.DeleteObjectAndConfirm(ctx, t, c, expectedCRD(suffix))) + testutils.CreateThenCleanup(ctx, t, c, constraintTemplate) clientset := kubernetes.NewForConfigOrDie(cfg) - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { crd := &apiextensionsv1.CustomResourceDefinition{} @@ -239,11 +191,11 @@ func TestReconcile(t *testing.T) { constraintTemplate := makeReconcileConstraintTemplate(suffix) cstr := newDenyAllCstr(suffix) - t.Cleanup(deleteObjectAndConfirm(ctx, t, c, cstr)) - t.Cleanup(deleteObjectAndConfirm(ctx, t, c, expectedCRD(suffix))) - createThenCleanup(ctx, t, c, constraintTemplate) + t.Cleanup(testutils.DeleteObjectAndConfirm(ctx, t, c, cstr)) + t.Cleanup(testutils.DeleteObjectAndConfirm(ctx, t, c, expectedCRD(suffix))) + testutils.CreateThenCleanup(ctx, t, c, constraintTemplate) - err = retry.OnError(constantRetry, func(error) bool { + err = retry.OnError(testutils.ConstantRetry, func(error) bool { return true }, func() error { return c.Create(ctx, cstr) @@ -293,12 +245,12 @@ func TestReconcile(t *testing.T) { constraintTemplate := makeReconcileConstraintTemplate(suffix) cstr := newDenyAllCstr(suffix) - t.Cleanup(deleteObjectAndConfirm(ctx, t, c, cstr)) - t.Cleanup(deleteObjectAndConfirm(ctx, t, c, expectedCRD(suffix))) - createThenCleanup(ctx, t, c, constraintTemplate) + t.Cleanup(testutils.DeleteObjectAndConfirm(ctx, t, c, cstr)) + t.Cleanup(testutils.DeleteObjectAndConfirm(ctx, t, c, expectedCRD(suffix))) + testutils.CreateThenCleanup(ctx, t, c, constraintTemplate) var crd *apiextensionsv1.CustomResourceDefinition - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { crd = &apiextensionsv1.CustomResourceDefinition{} @@ -315,7 +267,7 @@ func TestReconcile(t *testing.T) { t.Fatal(err) } - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { crd := &apiextensionsv1.CustomResourceDefinition{} @@ -339,7 +291,7 @@ func TestReconcile(t *testing.T) { t.Fatal(err) } - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { sList := &statusv1beta1.ConstraintPodStatusList{} @@ -355,7 +307,7 @@ func TestReconcile(t *testing.T) { t.Fatal(err) } - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { return c.Create(ctx, newDenyAllCstr(suffix)) @@ -409,7 +361,7 @@ func TestReconcile(t *testing.T) { // https://github.com/open-policy-agent/gatekeeper/pull/1595#discussion_r722819552 t.Cleanup(testutils.DeleteObject(t, c, instanceInvalidRego)) - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { ct := &v1beta1.ConstraintTemplate{} @@ -484,7 +436,7 @@ func TestReconcile(t *testing.T) { t.Fatal(err) } - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { resp, err := opaClient.Review(ctx, req) @@ -508,7 +460,7 @@ func TestReconcile_DeleteConstraintResources(t *testing.T) { logger.Info("Running test: Cancel the expectations when constraint gets deleted") // Setup the Manager - mgr, wm := setupManager(t) + mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) // start manager that will start tracker and controller @@ -573,13 +525,13 @@ violation[{"msg": "denied!"}] { // creating the gatekeeper-system namespace is necessary because that's where // status resources live by default - err = createGatekeeperNamespace(mgr.GetConfig()) + err = testutils.CreateGatekeeperNamespace(mgr.GetConfig()) if err != nil { t.Fatal(err) } // Set up tracker - tracker, err := readiness.SetupTrackerNoReadyz(mgr, false, false) + tracker, err := readiness.SetupTrackerNoReadyz(mgr, false, false, false) if err != nil { t.Fatal(err) } @@ -622,7 +574,7 @@ violation[{"msg": "denied!"}] { t.Fatalf("unexpected tracker, got %T", ot) } // ensure that expectations are set for the constraint gvk - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { gotExpected := tr.IsExpecting(gvk, types.NamespacedName{Name: "denyallconstraint"}) @@ -648,7 +600,7 @@ violation[{"msg": "denied!"}] { } // Check readiness tracker is satisfied post-reconcile - err = retry.OnError(constantRetry, func(err error) bool { + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { satisfied := tracker.For(gvk).Satisfied() @@ -663,7 +615,7 @@ violation[{"msg": "denied!"}] { } func constraintEnforced(ctx context.Context, c client.Client, suffix string) error { - return retry.OnError(constantRetry, func(err error) bool { + return retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { cstr := newDenyAllCstr(suffix) @@ -778,7 +730,7 @@ func applyCRD(ctx context.Context, client client.Client, gvk schema.GroupVersion u := &unstructured.UnstructuredList{} u.SetGroupVersionKind(gvk) - return retry.OnError(constantRetry, func(err error) bool { + return retry.OnError(testutils.ConstantRetry, func(err error) bool { return true }, func() error { if ctx.Err() != nil { @@ -788,99 +740,7 @@ func applyCRD(ctx context.Context, client client.Client, gvk schema.GroupVersion }) } -// deleteObjectAndConfirm returns a callback which deletes obj from the passed -// Client. Does result in mutations to obj. The callback includes a cached copy -// of all information required to delete obj in the callback, so it is safe to -// mutate obj afterwards. Similarly - client.Delete mutates its input, but -// the callback does not call client.Delete on obj. Instead, it creates a -// single-purpose Unstructured for this purpose. Thus, obj is not mutated after -// the callback is run. -func deleteObjectAndConfirm(ctx context.Context, t *testing.T, c client.Client, obj client.Object) func() { - t.Helper() - - // Cache the identifying information from obj. We refer to this cached - // information in the callback, and not obj itself. - gvk := obj.GetObjectKind().GroupVersionKind() - namespace := obj.GetNamespace() - name := obj.GetName() - - if gvk.Empty() { - // We can't send a proper delete request with an Unstructured without - // filling in GVK. The alternative would be to require tests to construct - // a valid Scheme or provide a factory method for the type to delete - this - // is easier. - t.Fatalf("gvk for %v/%v %T is empty", - namespace, name, obj) - } - - return func() { - t.Helper() - - // Construct a single-use Unstructured to send the Delete request. - toDelete := makeUnstructured(gvk, namespace, name) - err := c.Delete(ctx, toDelete) - if apierrors.IsNotFound(err) { - return - } else if err != nil { - t.Fatal(err) - } - - err = retry.OnError(constantRetry, func(err error) bool { - return true - }, func() error { - // Construct a single-use Unstructured to send the Get request. It isn't - // safe to reuse Unstructureds for each retry as Get modifies its input. - toGet := makeUnstructured(gvk, namespace, name) - key := client.ObjectKey{Namespace: namespace, Name: name} - err2 := c.Get(ctx, key, toGet) - if apierrors.IsGone(err2) || apierrors.IsNotFound(err2) { - return nil - } - - // Marshal the currently-gotten object, so it can be printed in test - // failure output. - s, _ := json.MarshalIndent(toGet, "", " ") - return fmt.Errorf("found %v %v:\n%s", gvk, key, string(s)) - }) - - if err != nil { - t.Fatal(err) - } - } -} - // This interface is getting used by tests to check the private objects of objectTracker. type testExpectations interface { IsExpecting(gvk schema.GroupVersionKind, nsName types.NamespacedName) bool } - -// createThenCleanup creates obj in Client, and then registers obj to be deleted -// at the end of the test. The passed obj is safely deepcopied before being -// passed to client.Create, so it is not mutated by this call. -func createThenCleanup(ctx context.Context, t *testing.T, c client.Client, obj client.Object) { - t.Helper() - cpy := obj.DeepCopyObject() - cpyObj, ok := cpy.(client.Object) - if !ok { - t.Fatalf("got obj.DeepCopyObject() type = %T, want %T", cpy, client.Object(nil)) - } - - err := c.Create(ctx, cpyObj) - if err != nil { - t.Fatal(err) - } - - // It is unnecessary to deepcopy obj as deleteObjectAndConfirm does not pass - // obj to any Client calls. - t.Cleanup(deleteObjectAndConfirm(ctx, t, c, obj)) -} - -func makeUnstructured(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { - u := &unstructured.Unstructured{ - Object: make(map[string]interface{}), - } - u.SetGroupVersionKind(gvk) - u.SetNamespace(namespace) - u.SetName(name) - return u -} diff --git a/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_suite_test.go b/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_suite_test.go index 64ea69a73ef..8fb12e0d951 100644 --- a/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_suite_test.go +++ b/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_suite_test.go @@ -16,65 +16,14 @@ limitations under the License. package constrainttemplatestatus_test import ( - "context" - stdlog "log" - "os" - "path/filepath" "testing" - "github.com/open-policy-agent/gatekeeper/apis" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/scheme" + "github.com/open-policy-agent/gatekeeper/test/testutils" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/envtest" ) var cfg *rest.Config func TestMain(m *testing.M) { - var err error - - t := &envtest.Environment{ - CRDDirectoryPaths: []string{ - filepath.Join("..", "..", "..", "vendor", "github.com", "open-policy-agent", "frameworks", "constraint", "deploy", "crds.yaml"), - filepath.Join("..", "..", "..", "config", "crd", "bases"), - }, - ErrorIfCRDPathMissing: true, - } - if err := apis.AddToScheme(scheme.Scheme); err != nil { - stdlog.Fatal(err) - } - - if cfg, err = t.Start(); err != nil { - stdlog.Fatal(err) - } - stdlog.Print("STARTED") - - code := m.Run() - if err = t.Stop(); err != nil { - stdlog.Printf("error while trying to stop server: %v", err) - } - os.Exit(code) -} - -// Bootstrap the gatekeeper-system namespace for use in tests. -func createGatekeeperNamespace(cfg *rest.Config) error { - c, err := client.New(cfg, client.Options{}) - if err != nil { - return err - } - - // Create gatekeeper namespace - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gatekeeper-system", - }, - } - - ctx := context.Background() - _, err = controllerutil.CreateOrUpdate(ctx, c, ns, func() error { return nil }) - return err + testutils.StartControlPlane(m, &cfg, 3) } diff --git a/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_test.go b/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_test.go index d7618b95243..dbb86abb500 100644 --- a/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_test.go +++ b/pkg/controller/constrainttemplatestatus/constrainttemplatestatus_controller_test.go @@ -108,7 +108,7 @@ violation[{"msg": "denied!"}] { // creating the gatekeeper-system namespace is necessary because that's where // status resources live by default - if err := createGatekeeperNamespace(mgr.GetConfig()); err != nil { + if err := testutils.CreateGatekeeperNamespace(mgr.GetConfig()); err != nil { t.Fatalf("want createGatekeeperNamespace(mgr.GetConfig()) error = nil, got %v", err) } @@ -126,7 +126,7 @@ violation[{"msg": "denied!"}] { testutils.Setenv(t, "POD_NAME", "no-pod") cs := watch.NewSwitch() - tracker, err := readiness.SetupTracker(mgr, false, false) + tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { t.Fatal(err) } diff --git a/pkg/controller/expansion/expansion_controller.go b/pkg/controller/expansion/expansion_controller.go index 6a1d0cf6aa0..be462937a65 100644 --- a/pkg/controller/expansion/expansion_controller.go +++ b/pkg/controller/expansion/expansion_controller.go @@ -2,18 +2,24 @@ package expansion import ( "context" - "flag" + "fmt" constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" "github.com/open-policy-agent/gatekeeper/apis/expansion/unversioned" "github.com/open-policy-agent/gatekeeper/apis/expansion/v1alpha1" + "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" + statusv1beta1 "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" "github.com/open-policy-agent/gatekeeper/pkg/expansion" "github.com/open-policy-agent/gatekeeper/pkg/logging" + "github.com/open-policy-agent/gatekeeper/pkg/metrics" "github.com/open-policy-agent/gatekeeper/pkg/mutation" "github.com/open-policy-agent/gatekeeper/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/pkg/util" "github.com/open-policy-agent/gatekeeper/pkg/watch" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -25,22 +31,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) -var ( - expansionEnabled = flag.Bool("enable-generator-resource-expansion", false, "(alpha) Enable the expansion of generator resources") - - log = logf.Log.WithName("controller").WithValues("kind", "ExpansionTemplate", logging.Process, "template_expansion_controller") -) +var log = logf.Log.WithName("controller").WithValues("kind", "ExpansionTemplate", logging.Process, "template_expansion_controller") type Adder struct { WatchManager *watch.Manager ExpansionSystem *expansion.System + Tracker *readiness.Tracker + // GetPod returns an instance of the currently running Gatekeeper pod + GetPod func(context.Context) (*corev1.Pod, error) } func (a *Adder) Add(mgr manager.Manager) error { - if !*expansionEnabled { + if !*expansion.ExpansionEnabled { return nil } - r := newReconciler(mgr, a.ExpansionSystem) + r := newReconciler(mgr, a.ExpansionSystem, a.GetPod, a.Tracker) return add(mgr, r) } @@ -50,7 +55,9 @@ func (a *Adder) InjectWatchManager(_ *watch.Manager) {} func (a *Adder) InjectControllerSwitch(_ *watch.ControllerSwitch) {} -func (a *Adder) InjectTracker(_ *readiness.Tracker) {} +func (a *Adder) InjectTracker(tracker *readiness.Tracker) { + a.Tracker = tracker +} func (a *Adder) InjectMutationSystem(_ *mutation.System) {} @@ -58,21 +65,32 @@ func (a *Adder) InjectExpansionSystem(expansionSystem *expansion.System) { a.ExpansionSystem = expansionSystem } +func (a *Adder) InjectGetPod(getPod func(ctx context.Context) (*corev1.Pod, error)) { + a.GetPod = getPod +} + func (a *Adder) InjectProviderCache(_ *externaldata.ProviderCache) {} type Reconciler struct { client.Client - system *expansion.System - scheme *runtime.Scheme - registry *etRegistry + system *expansion.System + scheme *runtime.Scheme + registry *etRegistry + statusClient client.StatusClient + tracker *readiness.Tracker + + getPod func(context.Context) (*corev1.Pod, error) } -func newReconciler(mgr manager.Manager, system *expansion.System) *Reconciler { +func newReconciler(mgr manager.Manager, system *expansion.System, getPod func(ctx context.Context) (*corev1.Pod, error), tracker *readiness.Tracker) *Reconciler { return &Reconciler{ - Client: mgr.GetClient(), - system: system, - scheme: mgr.GetScheme(), - registry: newRegistry(), + Client: mgr.GetClient(), + system: system, + scheme: mgr.GetScheme(), + registry: newRegistry(), + statusClient: mgr.GetClient(), + getPod: getPod, + tracker: tracker, } } @@ -88,11 +106,12 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + defer r.registry.report(ctx) log.Info("Reconcile", "request", request, "namespace", request.Namespace, "name", request.Name) deleted := false - te := &v1alpha1.ExpansionTemplate{} - err := r.Get(ctx, request.NamespacedName, te) + versionedET := &v1alpha1.ExpansionTemplate{} + err := r.Get(ctx, request.NamespacedName, versionedET) if err != nil { if !errors.IsNotFound(err) { return reconcile.Result{}, err @@ -100,34 +119,113 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( deleted = true } - unversionedTE := &unversioned.ExpansionTemplate{} - if err := r.scheme.Convert(te, unversionedTE, nil); err != nil { + et := &unversioned.ExpansionTemplate{} + if err := r.scheme.Convert(versionedET, et, nil); err != nil { return reconcile.Result{}, err } - nsName := types.NamespacedName{ - Namespace: unversionedTE.GetNamespace(), - Name: unversionedTE.GetName(), - } + if deleted { - // unversionedTE will be an empty struct. We set the metadata name, which is + // et will be an empty struct. We set the metadata name, which is // used as a key to delete it from the expansion system - unversionedTE.ObjectMeta.Name = request.Name - if err := r.system.RemoveTemplate(unversionedTE); err != nil { + et.Name = request.Name + if err := r.system.RemoveTemplate(et); err != nil { + r.getTracker().TryCancelExpect(versionedET) return reconcile.Result{}, err } - log.Info("removed template expansion", "template name", unversionedTE.ObjectMeta.Name) - r.registry.remove(nsName) + log.Info("removed expansion template", "template name", et.GetName()) + r.registry.remove(request.NamespacedName) + r.getTracker().CancelExpect(versionedET) + return reconcile.Result{}, r.deleteStatus(ctx, request.NamespacedName.Name) + } + + upsertErr := r.system.UpsertTemplate(et) + if upsertErr == nil { + log.Info("[readiness] observed ExpansionTemplate", "template name", et.GetName()) + r.getTracker().Observe(versionedET) + r.registry.add(request.NamespacedName, metrics.ActiveStatus) } else { - if err := r.system.UpsertTemplate(unversionedTE); err != nil { - return reconcile.Result{}, err + r.getTracker().TryCancelExpect(versionedET) + r.registry.add(request.NamespacedName, metrics.ErrorStatus) + log.Error(upsertErr, "upserting template", "template_name", et.GetName()) + } + + return reconcile.Result{}, r.updateOrCreatePodStatus(ctx, et, upsertErr) +} + +func (r *Reconciler) deleteStatus(ctx context.Context, etName string) error { + status := &v1beta1.ExpansionTemplatePodStatus{} + pod, err := r.getPod(ctx) + if err != nil { + return fmt.Errorf("getting reconciler pod: %w", err) + } + sName, err := v1beta1.KeyForExpansionTemplate(pod.Name, etName) + if err != nil { + return fmt.Errorf("getting key for expansiontemplate: %w", err) + } + status.SetName(sName) + status.SetNamespace(util.GetNamespace()) + if err := r.Delete(ctx, status); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func (r *Reconciler) updateOrCreatePodStatus(ctx context.Context, et *unversioned.ExpansionTemplate, etErr error) error { + pod, err := r.getPod(ctx) + if err != nil { + return fmt.Errorf("getting reconciler pod: %w", err) + } + + // Check if it exists already + sNS := pod.Namespace + sName, err := v1beta1.KeyForExpansionTemplate(pod.Name, et.GetName()) + if err != nil { + return fmt.Errorf("getting key for expansiontemplate: %w", err) + } + shouldCreate := true + status := &v1beta1.ExpansionTemplatePodStatus{} + + err = r.Get(ctx, types.NamespacedName{Namespace: sNS, Name: sName}, status) + switch { + case err == nil: + shouldCreate = false + case apierrors.IsNotFound(err): + if status, err = r.newETStatus(pod, et); err != nil { + return fmt.Errorf("creating new expansiontemplate status: %w", err) } - log.Info("upserted template expansion", "template name", unversionedTE.ObjectMeta.Name) - r.registry.add(nsName) + default: + return fmt.Errorf("getting expansion status in name %s, namespace %s: %w", et.GetName(), et.GetNamespace(), err) } - if err := r.registry.report(ctx); err != nil { - log.Error(err, "error reporting template expansion metrics", "namespacedName", nsName) + setStatusError(status, etErr) + status.Status.ObservedGeneration = et.GetGeneration() + + if shouldCreate { + return r.Create(ctx, status) + } + return r.Update(ctx, status) +} + +func (r *Reconciler) newETStatus(pod *corev1.Pod, et *unversioned.ExpansionTemplate) (*v1beta1.ExpansionTemplatePodStatus, error) { + status, err := statusv1beta1.NewExpansionTemplateStatusForPod(pod, et.GetName(), r.scheme) + if err != nil { + return nil, fmt.Errorf("creating status for pod: %w", err) + } + status.Status.TemplateUID = et.GetUID() + + return status, nil +} + +func (r *Reconciler) getTracker() readiness.Expectations { + return r.tracker.For(v1alpha1.GroupVersion.WithKind("ExpansionTemplate")) +} + +func setStatusError(status *v1beta1.ExpansionTemplatePodStatus, etErr error) { + if etErr == nil { + status.Status.Errors = nil + return } - return reconcile.Result{}, nil + e := &v1beta1.ExpansionTemplateError{Message: etErr.Error()} + status.Status.Errors = append(status.Status.Errors, e) } diff --git a/pkg/controller/expansion/expansion_controller_test.go b/pkg/controller/expansion/expansion_controller_test.go new file mode 100644 index 00000000000..802030dc40d --- /dev/null +++ b/pkg/controller/expansion/expansion_controller_test.go @@ -0,0 +1,203 @@ +package expansion + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/open-policy-agent/gatekeeper/apis/expansion/v1alpha1" + statusv1beta1 "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" + "github.com/open-policy-agent/gatekeeper/pkg/expansion" + "github.com/open-policy-agent/gatekeeper/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/pkg/mutation" + "github.com/open-policy-agent/gatekeeper/pkg/mutation/match" + "github.com/open-policy-agent/gatekeeper/pkg/readiness" + testclient "github.com/open-policy-agent/gatekeeper/test/clients" + "github.com/open-policy-agent/gatekeeper/test/testutils" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/retry" +) + +var cfg *rest.Config + +func TestMain(m *testing.M) { + testutils.StartControlPlane(m, &cfg, 3) +} + +func TestReconcile(t *testing.T) { + // Uncommenting the below enables logging of K8s internals like watch. + // fs := flag.NewFlagSet("", flag.PanicOnError) + // klog.InitFlags(fs) + // fs.Parse([]string{"--alsologtostderr", "-v=10"}) + // klog.SetOutput(os.Stderr) + + mgr, _ := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + + // creating the gatekeeper-system namespace is necessary because that's where + // status resources live by default + err := testutils.CreateGatekeeperNamespace(mgr.GetConfig()) + if err != nil { + t.Fatal(err) + } + + mutSystem := mutation.NewSystem(mutation.SystemOpts{}) + expSystem := expansion.NewSystem(mutSystem) + + testutils.Setenv(t, "POD_NAME", "no-pod") + + tracker, err := readiness.SetupTracker(mgr, false, false, true) + if err != nil { + t.Fatal(err) + } + + pod := fakes.Pod( + fakes.WithNamespace("gatekeeper-system"), + fakes.WithName("no-pod"), + ) + + r := newReconciler(mgr, expSystem, func(context.Context) (*corev1.Pod, error) { return pod, nil }, tracker) + if err != nil { + t.Fatal(err) + } + + err = add(mgr, r) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + testutils.StartManager(ctx, t, mgr) + + t.Run("creating ET creates status obj, deleting ET deletes status", func(t *testing.T) { + t.Log("running test: creating ET creates ETPodStatus, deleting ET deletes status") + + etName := "default-et" + et := newET(etName) + + sName, err := statusv1beta1.KeyForExpansionTemplate("no-pod", etName) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(testutils.DeleteObjectAndConfirm(ctx, t, c, et)) + testutils.CreateThenCleanup(ctx, t, c, et) + + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { + return true + }, func() error { + // First, get the ET + et := &v1alpha1.ExpansionTemplate{} + nsName := types.NamespacedName{Name: etName} + if err := c.Get(ctx, nsName, et); err != nil { + return err + } + if err != nil { + return fmt.Errorf("error fetching ET: %w", err) + } + + // Get the ETPodStatus + status := &statusv1beta1.ExpansionTemplatePodStatus{} + nsName = types.NamespacedName{ + Name: sName, + Namespace: "gatekeeper-system", + } + if err := c.Get(ctx, nsName, status); err != nil { + return err + } + if err != nil { + return fmt.Errorf("error fetching ET status: %w", err) + } + if status.Status.TemplateUID == et.GetUID() { + return nil + } + return fmt.Errorf("ExpansionTemplatePodStatus.Status.TemplateUID %q does not match ExpansionTemplate.GetUID() %q", status.Status.TemplateUID, et.GetUID()) + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Delete(ctx, et); err != nil { + t.Fatalf("error deleting ET: %s", err) + } + + err = retry.OnError(testutils.ConstantRetry, func(err error) bool { + return true + }, func() error { + // Get the ETPodStatus + status := &statusv1beta1.ExpansionTemplatePodStatus{} + nsName := types.NamespacedName{ + Name: sName, + Namespace: "gatekeeper-system", + } + if err := c.Get(ctx, nsName, status); err != nil && apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("expected IsNotFound when fetching status, but got: %w", err) + }) + if err != nil { + t.Fatal(err) + } + }) +} + +func TestAddStatusError(t *testing.T) { + tests := []struct { + name string + inputStatus statusv1beta1.ExpansionTemplatePodStatus + etErr error + wantStatus statusv1beta1.ExpansionTemplatePodStatus + }{ + { + name: "no err", + inputStatus: statusv1beta1.ExpansionTemplatePodStatus{}, + etErr: nil, + wantStatus: statusv1beta1.ExpansionTemplatePodStatus{Status: statusv1beta1.ExpansionTemplatePodStatusStatus{}}, + }, + { + name: "with err", + inputStatus: statusv1beta1.ExpansionTemplatePodStatus{}, + etErr: errors.New("big problem"), + wantStatus: statusv1beta1.ExpansionTemplatePodStatus{ + Status: statusv1beta1.ExpansionTemplatePodStatusStatus{ + Errors: []*statusv1beta1.ExpansionTemplateError{{Message: "big problem"}}, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + setStatusError(&tc.inputStatus, tc.etErr) + if diff := cmp.Diff(tc.inputStatus, tc.wantStatus); diff != "" { + t.Errorf("got: %v\nwant: %v\ndiff: %s", tc.inputStatus, tc.wantStatus, diff) + } + }) + } +} + +func newET(name string) *v1alpha1.ExpansionTemplate { + et := &v1alpha1.ExpansionTemplate{ + ObjectMeta: v1.ObjectMeta{Name: name}, + Spec: v1alpha1.ExpansionTemplateSpec{ + ApplyTo: []match.ApplyTo{{ + Groups: []string{"apps"}, + Kinds: []string{"Deployment"}, + Versions: []string{"v1"}, + }}, + TemplateSource: "spec.template", + GeneratedGVK: v1alpha1.GeneratedGVK{ + Kind: "Pod", + Version: "v1", + }, + }, + } + et.SetGroupVersionKind(v1alpha1.GroupVersion.WithKind("ExpansionTemplate")) + return et +} diff --git a/pkg/controller/expansion/stats_reporter.go b/pkg/controller/expansion/stats_reporter.go index 91bf1c45076..a6695214d71 100644 --- a/pkg/controller/expansion/stats_reporter.go +++ b/pkg/controller/expansion/stats_reporter.go @@ -6,6 +6,7 @@ import ( "github.com/open-policy-agent/gatekeeper/pkg/metrics" "go.opencensus.io/stats" "go.opencensus.io/stats/view" + "go.opencensus.io/tag" "k8s.io/apimachinery/pkg/types" ) @@ -15,7 +16,8 @@ const ( ) var ( - etM = stats.Int64(etMetricName, etDesc, stats.UnitDimensionless) + etM = stats.Int64(etMetricName, etDesc, stats.UnitDimensionless) + statusKey = tag.MustNewKey("status") views = []*view.View{ { @@ -23,6 +25,7 @@ var ( Measure: etM, Description: etDesc, Aggregation: view.LastValue(), + TagKeys: []tag.Key{statusKey}, }, } ) @@ -38,33 +41,50 @@ func register() error { } func newRegistry() *etRegistry { - return &etRegistry{cache: make(map[types.NamespacedName]bool)} + return &etRegistry{cache: make(map[types.NamespacedName]metrics.Status)} } type etRegistry struct { - cache map[types.NamespacedName]bool + cache map[types.NamespacedName]metrics.Status dirty bool } -func (r *etRegistry) add(key types.NamespacedName) { - r.cache[key] = true +func (r *etRegistry) add(key types.NamespacedName, status metrics.Status) { + v, ok := r.cache[key] + if ok && v == status { + return + } + r.cache[key] = status r.dirty = true } func (r *etRegistry) remove(key types.NamespacedName) { + if _, exists := r.cache[key]; !exists { + return + } delete(r.cache, key) r.dirty = true } -func (r *etRegistry) report(ctx context.Context) error { +func (r *etRegistry) report(ctx context.Context) { if !r.dirty { - return nil + return } - if err := metrics.Record(ctx, etM.M(int64(len(r.cache)))); err != nil { - r.dirty = false - return err + totals := make(map[metrics.Status]int64) + for _, status := range r.cache { + totals[status]++ } - return nil + for _, s := range metrics.AllStatuses { + ctx, err := tag.New(ctx, tag.Insert(statusKey, string(s))) + if err != nil { + log.Error(err, "failed to create status tag for expansion templates") + continue + } + err = metrics.Record(ctx, etM.M(totals[s])) + if err != nil { + log.Error(err, "failed to record total expansion templates") + } + } } diff --git a/pkg/controller/expansionstatus/expansionstatus_controller.go b/pkg/controller/expansionstatus/expansionstatus_controller.go new file mode 100644 index 00000000000..b22489b4810 --- /dev/null +++ b/pkg/controller/expansionstatus/expansionstatus_controller.go @@ -0,0 +1,213 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package expansionstatus + +import ( + "context" + "fmt" + "sort" + + "github.com/go-logr/logr" + constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" + "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" + expansionv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/expansion/v1alpha1" + "github.com/open-policy-agent/gatekeeper/apis/status/v1beta1" + "github.com/open-policy-agent/gatekeeper/pkg/expansion" + "github.com/open-policy-agent/gatekeeper/pkg/logging" + "github.com/open-policy-agent/gatekeeper/pkg/mutation" + "github.com/open-policy-agent/gatekeeper/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/pkg/util" + "github.com/open-policy-agent/gatekeeper/pkg/watch" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var log = logf.Log.WithName("controller").WithValues(logging.Process, "expansion_template_status_controller") + +type Adder struct { + Opa *constraintclient.Client + WatchManager *watch.Manager +} + +func (a *Adder) InjectOpa(o *constraintclient.Client) {} + +func (a *Adder) InjectWatchManager(w *watch.Manager) {} + +func (a *Adder) InjectControllerSwitch(cs *watch.ControllerSwitch) {} + +func (a *Adder) InjectTracker(t *readiness.Tracker) {} + +func (a *Adder) InjectMutationSystem(mutationSystem *mutation.System) {} + +func (a *Adder) InjectExpansionSystem(expansionSystem *expansion.System) {} + +func (a *Adder) InjectProviderCache(providerCache *externaldata.ProviderCache) {} + +// Add creates a new Constraint Status Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func (a *Adder) Add(mgr manager.Manager) error { + if !*expansion.ExpansionEnabled { + return nil + } + r := newReconciler(mgr) + return add(mgr, r) +} + +// newReconciler returns a new reconcile.Reconciler. +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileExpansionStatus{ + // Separate reader and writer because manager's default client bypasses the cache for unstructured resources. + writer: mgr.GetClient(), + statusClient: mgr.GetClient(), + reader: mgr.GetCache(), + + scheme: mgr.GetScheme(), + log: log, + } +} + +// PodStatusToExpansionTemplateMapper correlates a ExpansionTemplatePodStatus with its corresponding expansion template. +// `selfOnly` tells the mapper to only map statuses corresponding to the current pod. +func PodStatusToExpansionTemplateMapper(selfOnly bool) handler.MapFunc { + return func(obj client.Object) []reconcile.Request { + labels := obj.GetLabels() + name, ok := labels[v1beta1.ExpansionTemplateNameLabel] + if !ok { + log.Error(fmt.Errorf("expansion template status resource with no mapping label: %s", obj.GetName()), "missing label while attempting to map a expansion template status resource") + return nil + } + if selfOnly { + pod, ok := labels[v1beta1.PodLabel] + if !ok { + log.Error(fmt.Errorf("expansion template status resource with no pod label: %s", obj.GetName()), "missing label while attempting to map a expansion template status resource") + } + // Do not attempt to reconcile the resource when other pods have changed their status + if pod != util.GetPodName() { + return nil + } + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: name}}} + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler. +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("expansion-template-status-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to ExpansionTemplateStatus + err = c.Watch( + &source.Kind{Type: &v1beta1.ExpansionTemplatePodStatus{}}, + handler.EnqueueRequestsFromMapFunc(PodStatusToExpansionTemplateMapper(false)), + ) + if err != nil { + return err + } + + // Watch for changes to ExpansionTemplate + err = c.Watch(&source.Kind{Type: &expansionv1alpha1.ExpansionTemplate{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + return nil +} + +var _ reconcile.Reconciler = &ReconcileExpansionStatus{} + +// ReconcileExpansionStatus reconciles an arbitrary constraint object described by Kind. +type ReconcileExpansionStatus struct { + reader client.Reader + writer client.Writer + statusClient client.StatusClient + + scheme *runtime.Scheme + log logr.Logger +} + +// +kubebuilder:rbac:groups=expansion.gatekeeper.sh,resources=*,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=status.gatekeeper.sh,resources=*,verbs=get;list;watch;create;update;patch;delete + +// Reconcile reads that state of the cluster for a constraint object and makes changes based on the state read +// and what is in the constraint.Spec. +func (r *ReconcileExpansionStatus) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + et := &expansionv1alpha1.ExpansionTemplate{} + err := r.reader.Get(ctx, request.NamespacedName, et) + if err != nil { + // If the ExpansionTemplate does not exist then we are done + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + sObjs := &v1beta1.ExpansionTemplatePodStatusList{} + if err := r.reader.List( + ctx, + sObjs, + client.MatchingLabels{v1beta1.ExpansionTemplateNameLabel: request.Name}, + client.InNamespace(util.GetNamespace()), + ); err != nil { + return reconcile.Result{}, err + } + statusObjs := make(sortableStatuses, len(sObjs.Items)) + copy(statusObjs, sObjs.Items) + sort.Sort(statusObjs) + + var s []v1beta1.ExpansionTemplatePodStatusStatus + // created is true if at least one Pod hasn't reported any errors + + for i := range statusObjs { + // Don't report status if it's not for the correct object. This can happen + // if a watch gets interrupted, causing the constraint status to be deleted + // out from underneath it + if statusObjs[i].Status.TemplateUID != et.GetUID() { + continue + } + s = append(s, statusObjs[i].Status) + } + + et.Status.ByPod = s + + if err := r.statusClient.Status().Update(ctx, et); err != nil { + return reconcile.Result{Requeue: true}, nil + } + return reconcile.Result{}, nil +} + +type sortableStatuses []v1beta1.ExpansionTemplatePodStatus + +func (s sortableStatuses) Len() int { + return len(s) +} + +func (s sortableStatuses) Less(i, j int) bool { + return s[i].Status.ID < s[j].Status.ID +} + +func (s sortableStatuses) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} diff --git a/pkg/controller/externaldata/externaldata_controller_suite_test.go b/pkg/controller/externaldata/externaldata_controller_suite_test.go index 35169bc3357..d04987e6f7e 100644 --- a/pkg/controller/externaldata/externaldata_controller_suite_test.go +++ b/pkg/controller/externaldata/externaldata_controller_suite_test.go @@ -17,44 +17,17 @@ package externaldata import ( "context" - stdlog "log" - "os" - "path/filepath" "testing" - "github.com/open-policy-agent/gatekeeper/apis" - "k8s.io/client-go/kubernetes/scheme" + "github.com/open-policy-agent/gatekeeper/test/testutils" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var cfg *rest.Config func TestMain(m *testing.M) { - var err error - - t := &envtest.Environment{ - CRDDirectoryPaths: []string{ - filepath.Join("..", "..", "..", "vendor", "github.com", "open-policy-agent", "frameworks", "constraint", "deploy", "crds.yaml"), - filepath.Join("..", "..", "..", "config", "crd", "bases"), - }, - ErrorIfCRDPathMissing: true, - } - if err := apis.AddToScheme(scheme.Scheme); err != nil { - stdlog.Fatal(err) - } - - if cfg, err = t.Start(); err != nil { - stdlog.Fatal(err) - } - stdlog.Print("STARTED") - - code := m.Run() - if err = t.Stop(); err != nil { - stdlog.Printf("error while trying to stop server: %v", err) - } - os.Exit(code) + testutils.StartControlPlane(m, &cfg, 3) } // SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and diff --git a/pkg/controller/externaldata/externaldata_controller_test.go b/pkg/controller/externaldata/externaldata_controller_test.go index 717d6c813b3..f16adbefd2c 100644 --- a/pkg/controller/externaldata/externaldata_controller_test.go +++ b/pkg/controller/externaldata/externaldata_controller_test.go @@ -96,7 +96,7 @@ func TestReconcile(t *testing.T) { } cs := watch.NewSwitch() - tracker, err := readiness.SetupTracker(mgr, false, true) + tracker, err := readiness.SetupTracker(mgr, false, true, false) if err != nil { t.Fatal(err) } diff --git a/pkg/controller/mutators/core/controller_suite_test.go b/pkg/controller/mutators/core/controller_suite_test.go index 510170ad405..9e1314004d5 100644 --- a/pkg/controller/mutators/core/controller_suite_test.go +++ b/pkg/controller/mutators/core/controller_suite_test.go @@ -16,19 +16,14 @@ limitations under the License. package core import ( - "context" "log" "os" "path/filepath" "testing" "github.com/open-policy-agent/gatekeeper/apis" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/envtest" ) @@ -57,22 +52,3 @@ func TestMain(m *testing.M) { } os.Exit(code) } - -// Bootstrap the gatekeeper-system namespace for use in tests. -func createGatekeeperNamespace(cfg *rest.Config) error { - c, err := client.New(cfg, client.Options{}) - if err != nil { - return err - } - - // Create gatekeeper namespace - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gatekeeper-system", - }, - } - - ctx := context.Background() - _, err = controllerutil.CreateOrUpdate(ctx, c, ns, func() error { return nil }) - return err -} diff --git a/pkg/controller/mutators/core/controller_test.go b/pkg/controller/mutators/core/controller_test.go index 6e9985715f8..3a86e12dfa9 100644 --- a/pkg/controller/mutators/core/controller_test.go +++ b/pkg/controller/mutators/core/controller_test.go @@ -117,13 +117,13 @@ func TestReconcile(t *testing.T) { // creating the gatekeeper-system namespace is necessary because that's where // status resources live by default - if err := createGatekeeperNamespace(mgr.GetConfig()); err != nil { + if err := testutils.CreateGatekeeperNamespace(mgr.GetConfig()); err != nil { t.Fatalf("want createGatekeeperNamespace(mgr.GetConfig()) error = nil, got %v", err) } mSys := mutation.NewSystem(mutation.SystemOpts{}) - tracker, err := readiness.SetupTracker(mgr, true, false) + tracker, err := readiness.SetupTracker(mgr, true, false, false) if err != nil { t.Fatal(err) } diff --git a/pkg/expansion/system.go b/pkg/expansion/system.go index 0844f45c108..6984b247b96 100644 --- a/pkg/expansion/system.go +++ b/pkg/expansion/system.go @@ -2,6 +2,7 @@ package expansion import ( "encoding/json" + "flag" "fmt" "strings" "sync" @@ -14,6 +15,12 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) +var ExpansionEnabled *bool + +func init() { + ExpansionEnabled = flag.Bool("enable-generator-resource-expansion", false, "(alpha) Enable the expansion of generator resources") +} + type System struct { lock sync.RWMutex templates map[string]*expansionunversioned.ExpansionTemplate @@ -34,7 +41,7 @@ func (s *System) UpsertTemplate(template *expansionunversioned.ExpansionTemplate s.lock.Lock() defer s.lock.Unlock() - if err := validateTemplate(template); err != nil { + if err := ValidateTemplate(template); err != nil { return err } @@ -55,7 +62,7 @@ func (s *System) RemoveTemplate(template *expansionunversioned.ExpansionTemplate return nil } -func validateTemplate(template *expansionunversioned.ExpansionTemplate) error { +func ValidateTemplate(template *expansionunversioned.ExpansionTemplate) error { k := keyForTemplate(template) if k == "" { return fmt.Errorf("ExpansionTemplate has empty name field") diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index 07c89a51b4e..b35dc972588 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -26,6 +26,7 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" configv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1" + expansionv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/expansion/v1alpha1" mutationv1 "github.com/open-policy-agent/gatekeeper/apis/mutations/v1" mutationsv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/mutations/v1alpha1" "github.com/open-policy-agent/gatekeeper/pkg/keys" @@ -67,6 +68,7 @@ type Tracker struct { modifySet *objectTracker assignImage *objectTracker externalDataProvider *objectTracker + expansions *objectTracker constraints *trackerMap data *trackerMap @@ -75,14 +77,15 @@ type Tracker struct { statsEnabled syncutil.SyncBool mutationEnabled bool externalDataEnabled bool + expansionEnabled bool } // NewTracker creates a new Tracker and initializes the internal trackers. -func NewTracker(lister Lister, mutationEnabled bool, externalDataEnabled bool) *Tracker { - return newTracker(lister, mutationEnabled, externalDataEnabled, nil) +func NewTracker(lister Lister, mutationEnabled, externalDataEnabled, expansionEnabled bool) *Tracker { + return newTracker(lister, mutationEnabled, externalDataEnabled, expansionEnabled, nil) } -func newTracker(lister Lister, mutationEnabled bool, externalDataEnabled bool, fn objDataFactory) *Tracker { +func newTracker(lister Lister, mutationEnabled, externalDataEnabled, expansionEnabled bool, fn objDataFactory) *Tracker { tracker := Tracker{ lister: lister, templates: newObjTracker(v1beta1.SchemeGroupVersion.WithKind("ConstraintTemplate"), fn), @@ -94,6 +97,7 @@ func newTracker(lister Lister, mutationEnabled bool, externalDataEnabled bool, f mutationEnabled: mutationEnabled, externalDataEnabled: externalDataEnabled, + expansionEnabled: expansionEnabled, } if mutationEnabled { tracker.assignMetadata = newObjTracker(mutationv1.GroupVersion.WithKind("AssignMetadata"), fn) @@ -104,6 +108,9 @@ func newTracker(lister Lister, mutationEnabled bool, externalDataEnabled bool, f if externalDataEnabled { tracker.externalDataProvider = newObjTracker(externaldatav1beta1.SchemeGroupVersion.WithKind("Provider"), fn) } + if expansionEnabled { + tracker.expansions = newObjTracker(expansionv1alpha1.GroupVersion.WithKind("ExpansionTemplate"), fn) + } return &tracker } @@ -153,6 +160,11 @@ func (t *Tracker) For(gvk schema.GroupVersionKind) Expectations { return t.assignImage } return noopExpectations{} + case gvk.GroupVersion() == expansionv1alpha1.GroupVersion && gvk.Kind == "ExpansionTemplate": + if t.expansionEnabled { + return t.expansions + } + return noopExpectations{} } // Avoid new constraint trackers after templates have been populated. @@ -232,6 +244,13 @@ func (t *Tracker) Satisfied() bool { log.V(1).Info("all expectations satisfied", "tracker", "provider") } + if t.expansionEnabled { + if !t.expansions.Satisfied() { + return false + } + log.V(1).Info("all expectations satisfied", "tracker", "expansiontemplates") + } + if operations.HasValidationOperations() { if !t.templates.Satisfied() { return false @@ -294,6 +313,11 @@ func (t *Tracker) Run(ctx context.Context) error { return t.trackExternalDataProvider(gctx) }) } + if t.expansionEnabled { + grp.Go(func() error { + return t.trackExpansionTemplates(gctx) + }) + } if operations.HasValidationOperations() { grp.Go(func() error { return t.trackConstraintTemplates(gctx) @@ -567,6 +591,31 @@ func (t *Tracker) trackAssignImage(ctx context.Context) error { return nil } +func (t *Tracker) trackExpansionTemplates(ctx context.Context) error { + defer func() { + t.expansions.ExpectationsDone() + log.V(1).Info("ExpansionTemplate expectations populated") + _ = t.constraintTrackers.Wait() + }() + + if !t.expansionEnabled { + return nil + } + + expansionList := &expansionv1alpha1.ExpansionTemplateList{} + lister := retryLister(t.lister, retryAll) + if err := lister.List(ctx, expansionList); err != nil { + return fmt.Errorf("listing ExpansionTemplate: %w", err) + } + log.V(1).Info("setting expectations for ExpansionTemplate", "ExpansionTemplate Count", len(expansionList.Items)) + + for index := range expansionList.Items { + log.V(1).Info("expecting ExpansionTemplate", "name", expansionList.Items[index].GetName()) + t.expansions.Expect(&expansionList.Items[index]) + } + return nil +} + func (t *Tracker) trackExternalDataProvider(ctx context.Context) error { defer func() { t.externalDataProvider.ExpectationsDone() @@ -858,6 +907,9 @@ func (t *Tracker) statsPrinter(ctx context.Context) { if t.externalDataEnabled { logUnsatisfiedExternalDataProvider(t) } + if t.expansionEnabled { + logUnsatisfiedExpansions(t) + } } } @@ -885,6 +937,12 @@ func logUnsatisfiedAssignImage(t *Tracker) { } } +func logUnsatisfiedExpansions(t *Tracker) { + for _, et := range t.expansions.unsatisfied() { + log.Info("unsatisfied ExpansionTemplate", "name", et.namespacedName) + } +} + func logUnsatisfiedExternalDataProvider(t *Tracker) { for _, amKey := range t.externalDataProvider.unsatisfied() { log.Info("unsatisfied Provider", "name", amKey.namespacedName) diff --git a/pkg/readiness/ready_tracker_test.go b/pkg/readiness/ready_tracker_test.go index 4cc6771a015..40156ff101a 100644 --- a/pkg/readiness/ready_tracker_test.go +++ b/pkg/readiness/ready_tracker_test.go @@ -32,6 +32,7 @@ import ( frameworksexternaldata "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" "github.com/open-policy-agent/gatekeeper/pkg/controller" "github.com/open-policy-agent/gatekeeper/pkg/controller/config/process" + "github.com/open-policy-agent/gatekeeper/pkg/expansion" "github.com/open-policy-agent/gatekeeper/pkg/fakes" "github.com/open-policy-agent/gatekeeper/pkg/mutation" mutationtypes "github.com/open-policy-agent/gatekeeper/pkg/mutation/types" @@ -45,6 +46,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -108,9 +110,12 @@ func setupController( wm *watch.Manager, opa *constraintclient.Client, mutationSystem *mutation.System, + expansionSystem *expansion.System, providerCache *frameworksexternaldata.ProviderCache, ) error { - tracker, err := readiness.SetupTracker(mgr, mutationSystem != nil, providerCache != nil) + *expansion.ExpansionEnabled = expansionSystem != nil + + tracker, err := readiness.SetupTracker(mgr, mutationSystem != nil, providerCache != nil, expansionSystem != nil) if err != nil { return fmt.Errorf("setting up tracker: %w", err) } @@ -135,6 +140,7 @@ func setupController( GetPod: func(ctx context.Context) (*corev1.Pod, error) { return pod, nil }, ProcessExcluder: processExcluder, MutationSystem: mutationSystem, + ExpansionSystem: expansionSystem, ProviderCache: providerCache, WatchSet: watch.NewSet(), } @@ -158,9 +164,10 @@ func Test_AssignMetadata(t *testing.T) { opaClient := setupOpa(t) mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) + expansionSystem := expansion.NewSystem(mutationSystem) providerCache := frameworksexternaldata.NewCache() - if err := setupController(mgr, wm, opaClient, mutationSystem, providerCache); err != nil { + if err := setupController(mgr, wm, opaClient, mutationSystem, expansionSystem, providerCache); err != nil { t.Fatalf("setupControllers: %v", err) } @@ -201,9 +208,10 @@ func Test_ModifySet(t *testing.T) { opaClient := setupOpa(t) mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) + expansionSystem := expansion.NewSystem(mutationSystem) providerCache := frameworksexternaldata.NewCache() - if err := setupController(mgr, wm, opaClient, mutationSystem, providerCache); err != nil { + if err := setupController(mgr, wm, opaClient, mutationSystem, expansionSystem, providerCache); err != nil { t.Fatalf("setupControllers: %v", err) } @@ -242,9 +250,10 @@ func Test_AssignImage(t *testing.T) { opaClient := setupOpa(t) mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) + expansionSystem := expansion.NewSystem(mutationSystem) providerCache := frameworksexternaldata.NewCache() - if err := setupController(mgr, wm, opaClient, mutationSystem, providerCache); err != nil { + if err := setupController(mgr, wm, opaClient, mutationSystem, expansionSystem, providerCache); err != nil { t.Fatalf("setupControllers: %v", err) } @@ -283,9 +292,10 @@ func Test_Assign(t *testing.T) { opaClient := setupOpa(t) mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) + expansionSystem := expansion.NewSystem(mutationSystem) providerCache := frameworksexternaldata.NewCache() - if err := setupController(mgr, wm, opaClient, mutationSystem, providerCache); err != nil { + if err := setupController(mgr, wm, opaClient, mutationSystem, expansionSystem, providerCache); err != nil { t.Fatalf("setupControllers: %v", err) } @@ -308,6 +318,61 @@ func Test_Assign(t *testing.T) { } } +func Test_ExpansionTemplate(t *testing.T) { + g := gomega.NewWithT(t) + + testutils.Setenv(t, "POD_NAME", "no-pod") + + // Apply fixtures *before* the controllers are setup. + err := applyFixtures("testdata") + if err != nil { + t.Fatalf("applying fixtures: %v", err) + } + + // Wire up the rest. + mgr, wm := setupManager(t) + opaClient := setupOpa(t) + + mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) + expansionSystem := expansion.NewSystem(mutationSystem) + providerCache := frameworksexternaldata.NewCache() + + if err := setupController(mgr, wm, opaClient, mutationSystem, expansionSystem, providerCache); err != nil { + t.Fatalf("setupControllers: %v", err) + } + + ctx := context.Background() + testutils.StartManager(ctx, t, mgr) + + g.Eventually(func() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + return probeIsReady(ctx) + }, 20*time.Second, 1*time.Second).Should(gomega.BeTrue()) + + // Verify that the ExpansionTemplate is registered by expanding a demo deployment + // and checking that the resulting Pod is non-nil + deployment := makeDeployment("demo-deployment") + o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(deployment) + if err != nil { + panic(fmt.Errorf("error converting deployment to unstructured: %w", err)) + } + u := unstructured.Unstructured{Object: o} + m := mutationtypes.Mutable{ + Object: &u, + Namespace: testNS, + Username: "", + Source: "All", + } + res, err := expansionSystem.Expand(&m) + if err != nil { + panic(fmt.Errorf("error expanding: %w", err)) + } + if len(res) != 1 { + t.Fatal("expected generator to expand into 1 pod, but got 0 resultants") + } +} + func Test_Provider(t *testing.T) { g := gomega.NewWithT(t) @@ -331,6 +396,7 @@ func Test_Provider(t *testing.T) { wm, opaClient, mutation.NewSystem(mutation.SystemOpts{}), + nil, providerCache); err != nil { t.Fatalf("setupControllers: %v", err) } @@ -385,7 +451,7 @@ func Test_Tracker(t *testing.T) { opaClient := setupOpa(t) providerCache := frameworksexternaldata.NewCache() - if err := setupController(mgr, wm, opaClient, mutation.NewSystem(mutation.SystemOpts{}), providerCache); err != nil { + if err := setupController(mgr, wm, opaClient, mutation.NewSystem(mutation.SystemOpts{}), nil, providerCache); err != nil { t.Fatalf("setupControllers: %v", err) } @@ -483,7 +549,7 @@ func Test_Tracker_UnregisteredCachedData(t *testing.T) { opaClient := setupOpa(t) providerCache := frameworksexternaldata.NewCache() - if err := setupController(mgr, wm, opaClient, mutation.NewSystem(mutation.SystemOpts{}), providerCache); err != nil { + if err := setupController(mgr, wm, opaClient, mutation.NewSystem(mutation.SystemOpts{}), nil, providerCache); err != nil { t.Fatalf("setupControllers: %v", err) } @@ -523,7 +589,7 @@ func Test_CollectDeleted(t *testing.T) { lister: mgr.GetAPIReader(), namespace: "gatekeeper-system", } - tracker := readiness.NewTracker(lister, false, false) + tracker := readiness.NewTracker(lister, false, false, false) err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return tracker.Run(ctx) })) diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index b732400f321..bfdc38fc1e5 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -72,7 +72,7 @@ func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { g := gomega.NewWithT(t) l := dummyLister{} - rt := newTracker(l, false, false, func() objData { + rt := newTracker(l, false, false, false, func() objData { return objData{retries: 0} }) @@ -114,7 +114,7 @@ func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { g := gomega.NewWithT(t) l := dummyLister{} - rt := newTracker(l, false, false, func() objData { + rt := newTracker(l, false, false, false, func() objData { return objData{retries: 2} }) diff --git a/pkg/readiness/setup.go b/pkg/readiness/setup.go index 10353c18f63..47e2b3721ce 100644 --- a/pkg/readiness/setup.go +++ b/pkg/readiness/setup.go @@ -25,8 +25,8 @@ import ( // SetupTracker sets up a readiness tracker and registers it to run under control of the // provided Manager object. // NOTE: Must be called _before_ the manager is started. -func SetupTracker(mgr manager.Manager, mutationEnabled bool, externalDataEnabled bool) (*Tracker, error) { - tracker, err := SetupTrackerNoReadyz(mgr, mutationEnabled, externalDataEnabled) +func SetupTracker(mgr manager.Manager, mutationEnabled, externalDataEnabled, expansionEnabled bool) (*Tracker, error) { + tracker, err := SetupTrackerNoReadyz(mgr, mutationEnabled, externalDataEnabled, expansionEnabled) if err != nil { return nil, err } @@ -40,8 +40,8 @@ func SetupTracker(mgr manager.Manager, mutationEnabled bool, externalDataEnabled // SetupTrackerNoReadyz sets up a readiness tracker and registers it to run under control of the // provided Manager object without instantiating /readyz (used for testing). -func SetupTrackerNoReadyz(mgr manager.Manager, mutationEnabled bool, externalDataEnabled bool) (*Tracker, error) { - tracker := NewTracker(mgr.GetAPIReader(), mutationEnabled, externalDataEnabled) +func SetupTrackerNoReadyz(mgr manager.Manager, mutationEnabled, externalDataEnabled, expansionEnabled bool) (*Tracker, error) { + tracker := NewTracker(mgr.GetAPIReader(), mutationEnabled, externalDataEnabled, expansionEnabled) err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return tracker.Run(ctx) diff --git a/pkg/readiness/testdata/99-expansion-template.yaml b/pkg/readiness/testdata/99-expansion-template.yaml new file mode 100644 index 00000000000..716a48456d4 --- /dev/null +++ b/pkg/readiness/testdata/99-expansion-template.yaml @@ -0,0 +1,15 @@ +apiVersion: expansion.gatekeeper.sh/v1alpha1 +kind: ExpansionTemplate +metadata: + name: demo +spec: + applyTo: + - groups: [ "apps" ] + kinds: [ "Deployment", "ReplicaSet" ] + versions: [ "v1" ] + templateSource: "spec.template" + enforcementAction: "deny" + generatedGVK: + kind: "Pod" + group: "" + version: "v1" diff --git a/pkg/readiness/testdata_test.go b/pkg/readiness/testdata_test.go index c91145bc366..2c39835aa50 100644 --- a/pkg/readiness/testdata_test.go +++ b/pkg/readiness/testdata_test.go @@ -19,6 +19,8 @@ import ( externaldatav1beta1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/externaldata/v1beta1" "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" mutationsv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/mutations/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -63,6 +65,8 @@ var testProvider = []*externaldatav1beta1.Provider{ makeProvider("demo"), } +var testNS = makeNS("demo") + func makeTemplate(name string) *templates.ConstraintTemplate { return &templates.ConstraintTemplate{ ObjectMeta: metav1.ObjectMeta{ @@ -154,3 +158,39 @@ func makeProvider(name string) *externaldatav1beta1.Provider { }, } } + +func makeDeployment(name string) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:latest", + }, + }, + }, + }, + }, + } +} + +func makeNS(name string) *corev1.Namespace { + return &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } +} diff --git a/pkg/webhook/policy.go b/pkg/webhook/policy.go index 0159f716348..6466caf4b71 100644 --- a/pkg/webhook/policy.go +++ b/pkg/webhook/policy.go @@ -34,6 +34,7 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" rtypes "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/apis" + expansionunversioned "github.com/open-policy-agent/gatekeeper/apis/expansion/unversioned" mutationsunversioned "github.com/open-policy-agent/gatekeeper/apis/mutations/unversioned" "github.com/open-policy-agent/gatekeeper/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/pkg/expansion" @@ -322,6 +323,8 @@ func (h *validationHandler) validateGatekeeperResources(ctx context.Context, req switch { case gvk.Group == "templates.gatekeeper.sh" && gvk.Kind == "ConstraintTemplate": return h.validateTemplate(ctx, req) + case gvk.Group == "expansion.gatekeeper.sh" && gvk.Kind == "ExpansionTemplate": + return h.validateExpansionTemplate(req) case gvk.Group == "constraints.gatekeeper.sh": return h.validateConstraint(req) case gvk.Group == "config.gatekeeper.sh" && gvk.Kind == "Config": @@ -407,6 +410,23 @@ func (h *validationHandler) validateConstraint(req *admission.Request) (bool, er return false, nil } +func (h *validationHandler) validateExpansionTemplate(req *admission.Request) (bool, error) { + obj, _, err := deserializer.Decode(req.AdmissionRequest.Object.Raw, nil, nil) + if err != nil { + return false, err + } + unversioned := &expansionunversioned.ExpansionTemplate{} + if err := runtimeScheme.Convert(obj, unversioned, nil); err != nil { + return false, err + } + err = expansion.ValidateTemplate(unversioned) + if err != nil { + return true, err + } + + return false, nil +} + func (h *validationHandler) validateConfigResource(req *admission.Request) error { if req.Name != keys.Config.Name { return fmt.Errorf("config resource must have name 'config'") diff --git a/test/bats/test.bats b/test/bats/test.bats index 307540d715a..4a53e1f1186 100644 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -417,6 +417,12 @@ __expansion_audit_test() { run kubectl apply -f test/expansion/loadbalancers_must_have_env.yaml wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_enforced k8srequiredlabels loadbalancers-must-have-env" + # check status resource on expansion template + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get -f test/expansion/expand_deployments.yaml -ojson | jq -r -e '.status.byPod[0]'" + local temp_uid=$(kubectl get -f test/expansion/expand_deployments.yaml -o jsonpath='{.metadata.uid}') + local byPod_uid=$(kubectl get -f test/expansion/expand_deployments.yaml -o jsonpath='{.status.byPod[0].templateUID}') + assert_match ${temp_uid} ${byPod_uid} + # assert that creating deployment without 'env' label is rejected run kubectl apply -f test/expansion/deployment_no_label.yaml assert_failure diff --git a/test/testutils/controller.go b/test/testutils/controller.go new file mode 100644 index 00000000000..050dd1c8413 --- /dev/null +++ b/test/testutils/controller.go @@ -0,0 +1,179 @@ +package testutils + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "testing" + "time" + + "github.com/open-policy-agent/gatekeeper/apis" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +var ( + vendorCRDPath = []string{"vendor", "github.com", "open-policy-agent", "frameworks", "constraint", "deploy", "crds.yaml"} + gkCRDPath = []string{"config", "crd", "bases"} +) + +// ConstantRetry makes 3,000 attempts at a rate of 100 per second. Since this +// is a test instance and not a "real" cluster, this is fine and there's no need +// to increase the wait time each iteration. +var ConstantRetry = wait.Backoff{ + Steps: 3000, + Duration: 10 * time.Millisecond, +} + +// CreateGatekeeperNamespace bootstraps the gatekeeper-system namespace for use in tests. +func CreateGatekeeperNamespace(cfg *rest.Config) error { + c, err := client.New(cfg, client.Options{}) + if err != nil { + return err + } + + // Create gatekeeper namespace + ns := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gatekeeper-system", + }, + } + + ctx := context.Background() + _, err = controllerutil.CreateOrUpdate(ctx, c, ns, func() error { return nil }) + return err +} + +// DeleteObjectAndConfirm returns a callback which deletes obj from the passed +// Client. Does result in mutations to obj. The callback includes a cached copy +// of all information required to delete obj in the callback, so it is safe to +// mutate obj afterwards. Similarly - client.Delete mutates its input, but +// the callback does not call client.Delete on obj. Instead, it creates a +// single-purpose Unstructured for this purpose. Thus, obj is not mutated after +// the callback is run. +func DeleteObjectAndConfirm(ctx context.Context, t *testing.T, c client.Client, obj client.Object) func() { + t.Helper() + + // Cache the identifying information from obj. We refer to this cached + // information in the callback, and not obj itself. + gvk := obj.GetObjectKind().GroupVersionKind() + namespace := obj.GetNamespace() + name := obj.GetName() + + if gvk.Empty() { + // We can't send a proper delete request with an Unstructured without + // filling in GVK. The alternative would be to require tests to construct + // a valid Scheme or provide a factory method for the type to delete - this + // is easier. + t.Fatalf("gvk for %v/%v %T is empty", + namespace, name, obj) + } + + return func() { + t.Helper() + + // Construct a single-use Unstructured to send the Delete request. + toDelete := makeUnstructured(gvk, namespace, name) + err := c.Delete(ctx, toDelete) + if apierrors.IsNotFound(err) { + return + } else if err != nil { + t.Fatal(err) + } + + err = retry.OnError(ConstantRetry, func(err error) bool { + return true + }, func() error { + // Construct a single-use Unstructured to send the Get request. It isn't + // safe to reuse Unstructureds for each retry as Get modifies its input. + toGet := makeUnstructured(gvk, namespace, name) + key := client.ObjectKey{Namespace: namespace, Name: name} + err2 := c.Get(ctx, key, toGet) + if apierrors.IsGone(err2) || apierrors.IsNotFound(err2) { + return nil + } + + // Marshal the currently-gotten object, so it can be printed in test + // failure output. + s, _ := json.MarshalIndent(toGet, "", " ") + return fmt.Errorf("found %v %v:\n%s", gvk, key, string(s)) + }) + + if err != nil { + t.Fatal(err) + } + } +} + +func StartControlPlane(m *testing.M, cfg **rest.Config, testerDepth int) { + walkbacks := make([]string, testerDepth) + for i := 0; i < testerDepth; i++ { + walkbacks[i] = ".." + } + t := &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join(append(walkbacks, vendorCRDPath...)...), + filepath.Join(append(walkbacks, gkCRDPath...)...), + }, + ErrorIfCRDPathMissing: true, + } + if err := apis.AddToScheme(scheme.Scheme); err != nil { + log.Fatal(err) + } + + var err error + if *cfg, err = t.Start(); err != nil { + log.Fatal(err) + } + log.Print("STARTED") + + code := m.Run() + if err = t.Stop(); err != nil { + log.Printf("error while trying to stop server: %v", err) + } + os.Exit(code) +} + +// CreateThenCleanup creates obj in Client, and then registers obj to be deleted +// at the end of the test. The passed obj is safely deepcopied before being +// passed to client.Create, so it is not mutated by this call. +func CreateThenCleanup(ctx context.Context, t *testing.T, c client.Client, obj client.Object) { + t.Helper() + cpy := obj.DeepCopyObject() + cpyObj, ok := cpy.(client.Object) + if !ok { + t.Fatalf("got obj.DeepCopyObject() type = %T, want %T", cpy, client.Object(nil)) + } + + err := c.Create(ctx, cpyObj) + if err != nil { + t.Fatal(err) + } + + // It is unnecessary to deepcopy obj as deleteObjectAndConfirm does not pass + // obj to any Client calls. + t.Cleanup(DeleteObjectAndConfirm(ctx, t, c, obj)) +} + +func makeUnstructured(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{ + Object: make(map[string]interface{}), + } + u.SetGroupVersionKind(gvk) + u.SetNamespace(namespace) + u.SetName(name) + return u +} diff --git a/test/testutils/manager.go b/test/testutils/manager.go index bc5ded5b27a..b3b255c9027 100644 --- a/test/testutils/manager.go +++ b/test/testutils/manager.go @@ -5,7 +5,14 @@ import ( "sync" "testing" + "github.com/open-policy-agent/gatekeeper/pkg/watch" + "github.com/open-policy-agent/gatekeeper/third_party/sigs.k8s.io/controller-runtime/pkg/dynamiccache" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) // StartManager starts mgr. Registers a cleanup function to stop the manager at the completion of the test. @@ -30,3 +37,34 @@ func StartManager(ctx context.Context, t *testing.T, mgr manager.Manager) { } }) } + +// SetupManager sets up a controller-runtime manager with registered watch manager. +func SetupManager(t *testing.T, cfg *rest.Config) (manager.Manager, *watch.Manager) { + t.Helper() + + metrics.Registry = prometheus.NewRegistry() + mgr, err := manager.New(cfg, manager.Options{ + MetricsBindAddress: "0", + NewCache: dynamiccache.New, + MapperProvider: func(c *rest.Config) (meta.RESTMapper, error) { + return apiutil.NewDynamicRESTMapper(c) + }, + Logger: NewLogger(t), + }) + if err != nil { + t.Fatalf("setting up controller manager: %s", err) + } + c := mgr.GetCache() + dc, ok := c.(watch.RemovableCache) + if !ok { + t.Fatalf("expected dynamic cache, got: %T", c) + } + wm, err := watch.New(dc) + if err != nil { + t.Fatalf("could not create watch manager: %s", err) + } + if err := mgr.Add(wm); err != nil { + t.Fatalf("could not add watch manager to manager: %s", err) + } + return mgr, wm +}