Add support for emit k8s events for audit (#739)

* Add support for emit audit events Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com> * Update helm chart with new flags Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com> * Update event ref uuid and emit-admission-events flag Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com> * Add alpha feature to readme and manifest updates Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com> * Add alpha to flag doc Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com> * rm flag from deployment yaml Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com> Signed-off-by: Bryce Cronkite-Ratcliff <brycecr@gmail.com>
open-policy-agent · Jul 28, 2020 · ff8d084 · ff8d084
1 parent e27164c
commit ff8d084
Show file tree

Hide file tree

Showing 15 changed files with 112 additions and 36 deletions.
diff --git a/Makefile b/Makefile
@@ -32,6 +32,12 @@ MANAGER_IMAGE_PATCH := "apiVersion: apps/v1\
 \n      containers:\
 \n      - image: <your image file>\
 \n        name: manager\
+\n        args:\
+\n        - --port=8443\
+\n        - --logtostderr\
+\n        - --emit-admission-events\
+\n        - --exempt-namespace=gatekeeper-system\
+\n        - --operation=webhook\
 \n---\
 \napiVersion: apps/v1\
 \nkind: Deployment\
@@ -43,7 +49,12 @@ MANAGER_IMAGE_PATCH := "apiVersion: apps/v1\
 \n    spec:\
 \n      containers:\
 \n      - image: <your image file>\
-\n        name: auditcontainer"
+\n        name: auditcontainer\
+\n        args:\
+\n        - --emit-audit-events\
+\n        - --operation=audit\
+\n        - --operation=status\
+\n        - --logtostderr"
 
 
 FRAMEWORK_PACKAGE := github.com/open-policy-agent/frameworks/constraint
@@ -107,7 +118,7 @@ e2e-helm-deploy:
 	cd .staging/helm && tar -xvf helmbin.tar.gz
 	./.staging/helm/linux-amd64/helm init --wait --history-max=5
 	kubectl -n kube-system wait --for=condition=Ready pod -l name=tiller --timeout=300s
-	./.staging/helm/linux-amd64/helm install manifest_staging/charts/gatekeeper --name=tiger --set image.repository=${HELM_REPO} --set image.release=${HELM_RELEASE}
+	./.staging/helm/linux-amd64/helm install manifest_staging/charts/gatekeeper --name=tiger --set image.repository=${HELM_REPO} --set image.release=${HELM_RELEASE} --set emitAdmissionEvents=true --set emitAuditEvents=true
 
 # Build manager binary
 manager: generate fmt vet

diff --git a/cmd/build/helmify/kustomize-for-helm.yaml b/cmd/build/helmify/kustomize-for-helm.yaml
@@ -53,7 +53,7 @@ spec:
           args:
             - --port=8443
             - --logtostderr
-            - --emit-deny-events
+            - --emit-admission-events={{ .Values.emitAdmissionEvents }}
             - --log-level={{ .Values.logLevel }}
             - --exempt-namespace=gatekeeper-system
             - --operation=webhook
@@ -83,6 +83,7 @@ spec:
         - --log-level={{ .Values.logLevel }}
         - --constraint-violations-limit={{ .Values.constraintViolationsLimit }}
         - --audit-from-cache={{ .Values.auditFromCache }}
+        - --emit-audit-events={{ .Values.emitAuditEvents }}
         - --operation=audit
         - --operation=status
         - --logtostderr

diff --git a/cmd/build/helmify/static/README.md b/cmd/build/helmify/static/README.md
@@ -8,6 +8,8 @@
 | constraintViolationsLimit | The maximum # of audit violations reported on a constraint  | `20`                                                                      |
 | auditFromCache            | Take the roster of resources to audit from the OPA cache    | `false`                                                                   |
 | disableValidatingWebhook  | Disable ValidatingWebhook                                   | `false`                                                                   |
+| emitAdmissionEvents       | Emit K8s events in gatekeeper namespace for admission violations (alpha feature)| `false`                                                                   |
+| emitAuditEvents           | Emit K8s events in gatekeeper namespace for audit violations (alpha feature)| `false`                                                                   |
 | logLevel                  | Minimum log level                                           | `INFO`                                                                    |
 | image.pullPolicy          | The image pull policy                                       | `IfNotPresent`                                                            |
 | image.repository          | Image repository                                            | `openpolicyagent/gatekeeper`                                              |

diff --git a/cmd/build/helmify/static/values.yaml b/cmd/build/helmify/static/values.yaml
@@ -4,6 +4,8 @@ constraintViolationsLimit: 20
 auditFromCache: false
 disableValidatingWebhook: false
 logLevel: INFO
+emitAdmissionEvents: false
+emitAuditEvents: false
 image:
   repository: openpolicyagent/gatekeeper
   release: v3.1.0-beta.10

diff --git a/config/crd/bases/config.gatekeeper.sh_configs.yaml b/config/crd/bases/config.gatekeeper.sh_configs.yaml
@@ -48,6 +48,12 @@ spec:
                     type: array
                 type: object
               type: array
+            readiness:
+              description: Configuration for readiness tracker
+              properties:
+                statsEnabled:
+                  type: boolean
+              type: object
             sync:
               description: Configuration for syncing k8s objects
               properties:

diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -47,7 +47,6 @@ spec:
           args:
             - "--port=8443"
             - "--logtostderr"
-            - "--emit-deny-events"
             - "--exempt-namespace=gatekeeper-system"
             - "--operation=webhook"
           image: openpolicyagent/gatekeeper:v3.1.0-beta.10

diff --git a/manifest_staging/charts/gatekeeper/README.md b/manifest_staging/charts/gatekeeper/README.md
@@ -8,6 +8,8 @@
 | constraintViolationsLimit | The maximum # of audit violations reported on a constraint  | `20`                                                                      |
 | auditFromCache            | Take the roster of resources to audit from the OPA cache    | `false`                                                                   |
 | disableValidatingWebhook  | Disable ValidatingWebhook                                   | `false`                                                                   |
+| emitAdmissionEvents       | Emit K8s events in gatekeeper namespace for admission violations (alpha feature)| `false`                                                                   |
+| emitAuditEvents           | Emit K8s events in gatekeeper namespace for audit violations (alpha feature)| `false`                                                                   |
 | logLevel                  | Minimum log level                                           | `INFO`                                                                    |
 | image.pullPolicy          | The image pull policy                                       | `IfNotPresent`                                                            |
 | image.repository          | Image repository                                            | `openpolicyagent/gatekeeper`                                              |

diff --git a/manifest_staging/charts/gatekeeper/templates/config-customresourcedefinition.yaml b/manifest_staging/charts/gatekeeper/templates/config-customresourcedefinition.yaml
@@ -56,6 +56,12 @@ spec:
                     type: array
                 type: object
               type: array
+            readiness:
+              description: Configuration for readiness tracker
+              properties:
+                statsEnabled:
+                  type: boolean
+              type: object
             sync:
               description: Configuration for syncing k8s objects
               properties:

diff --git a/manifest_staging/charts/gatekeeper/templates/gatekeeper-audit-deployment.yaml b/manifest_staging/charts/gatekeeper/templates/gatekeeper-audit-deployment.yaml
@@ -41,6 +41,7 @@ spec:
         - --log-level={{ .Values.logLevel }}
         - --constraint-violations-limit={{ .Values.constraintViolationsLimit }}
         - --audit-from-cache={{ .Values.auditFromCache }}
+        - --emit-audit-events={{ .Values.emitAuditEvents }}
         - --operation=audit
         - --operation=status
         - --logtostderr

diff --git a/manifest_staging/charts/gatekeeper/templates/gatekeeper-controller-manager-deployment.yaml b/manifest_staging/charts/gatekeeper/templates/gatekeeper-controller-manager-deployment.yaml
@@ -51,7 +51,7 @@ spec:
       - args:
         - --port=8443
         - --logtostderr
-        - --emit-deny-events
+        - --emit-admission-events={{ .Values.emitAdmissionEvents }}
         - --log-level={{ .Values.logLevel }}
         - --exempt-namespace=gatekeeper-system
         - --operation=webhook

diff --git a/manifest_staging/charts/gatekeeper/values.yaml b/manifest_staging/charts/gatekeeper/values.yaml
@@ -4,6 +4,8 @@ constraintViolationsLimit: 20
 auditFromCache: false
 disableValidatingWebhook: false
 logLevel: INFO
+emitAdmissionEvents: false
+emitAuditEvents: false
 image:
   repository: openpolicyagent/gatekeeper
   release: v3.1.0-beta.10

diff --git a/manifest_staging/deploy/gatekeeper.yaml b/manifest_staging/deploy/gatekeeper.yaml
@@ -57,6 +57,12 @@ spec:
                     type: array
                 type: object
               type: array
+            readiness:
+              description: Configuration for readiness tracker
+              properties:
+                statsEnabled:
+                  type: boolean
+              type: object
             sync:
               description: Configuration for syncing k8s objects
               properties:
@@ -731,7 +737,6 @@ spec:
       - args:
         - --port=8443
         - --logtostderr
-        - --emit-deny-events
         - --exempt-namespace=gatekeeper-system
         - --operation=webhook
         command:

diff --git a/pkg/audit/manager.go b/pkg/audit/manager.go
@@ -24,6 +24,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -43,6 +47,7 @@ var (
 	auditInterval             = flag.Uint("audit-interval", defaultAuditInterval, "interval to run audit in seconds. defaulted to 60 secs if unspecified, 0 to disable ")
 	constraintViolationsLimit = flag.Uint("constraint-violations-limit", defaultConstraintViolationsLimit, "limit of number of violations per constraint. defaulted to 20 violations if unspecified ")
 	auditFromCache            = flag.Bool("audit-from-cache", false, "pull resources from OPA cache when auditing")
+	emitAuditEvents           = flag.Bool("emit-audit-events", false, "(alpha) emit Kubernetes events in gatekeeper namespace with detailed info for each violation from an audit")
 	emptyAuditResults         []auditResult
 )
 
@@ -58,6 +63,8 @@ type Manager struct {
 	reporter        *reporter
 	log             logr.Logger
 	processExcluder *process.Excluder
+	eventRecorder   record.EventRecorder
+	gkNamespace     string
 }
 
 type auditResult struct {
@@ -111,6 +118,12 @@ func New(ctx context.Context, mgr manager.Manager, opa *opa.Client, processExclu
 		log.Error(err, "StatsReporter could not start")
 		return nil, err
 	}
+	eventBroadcaster := record.NewBroadcaster()
+	kubeClient := kubernetes.NewForConfigOrDie(mgr.GetConfig())
+	eventBroadcaster.StartRecordingToSink(&clientcorev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
+	recorder := eventBroadcaster.NewRecorder(
+		scheme.Scheme,
+		corev1.EventSource{Component: "gatekeeper-audit"})
 
 	am := &Manager{
 		opa:             opa,
@@ -120,6 +133,8 @@ func New(ctx context.Context, mgr manager.Manager, opa *opa.Client, processExclu
 		ctx:             ctx,
 		reporter:        reporter,
 		processExcluder: processExcluder,
+		eventRecorder:   recorder,
+		gkNamespace:     util.GetNamespace(),
 	}
 	return am, nil
 }
@@ -183,7 +198,7 @@ func (am *Manager) audit(ctx context.Context) error {
 		am.log.Info("Audit discovery client results", "violations", len(res))
 	}
 
-	updateLists, totalViolationsPerConstraint, totalViolationsPerEnforcementAction, err := am.getUpdateListsFromAuditResponses(res)
+	updateLists, totalViolationsPerConstraint, totalViolationsPerEnforcementAction, err := am.getUpdateListsFromAuditResponses(res, timestamp)
 	if err != nil {
 		return err
 	}
@@ -343,7 +358,7 @@ func (am *Manager) getAllConstraintKinds() ([]schema.GroupVersionKind, error) {
 	return ret, nil
 }
 
-func (am *Manager) getUpdateListsFromAuditResponses(res []*constraintTypes.Result) (map[string][]auditResult, map[string]int64, map[util.EnforcementAction]int64, error) {
+func (am *Manager) getUpdateListsFromAuditResponses(res []*constraintTypes.Result, timestamp string) (map[string][]auditResult, map[string]int64, map[util.EnforcementAction]int64, error) {
 	updateLists := make(map[string][]auditResult)
 	totalViolationsPerConstraint := make(map[string]int64)
 	totalViolationsPerEnforcementAction := make(map[util.EnforcementAction]int64)
@@ -380,10 +395,14 @@ func (am *Manager) getUpdateListsFromAuditResponses(res []*constraintTypes.Resul
 			enforcementAction: enforcementAction,
 			constraint:        r.Constraint,
 		}
+
 		updateLists[selfLink] = append(updateLists[selfLink], result)
 		ea := util.EnforcementAction(enforcementAction)
 		totalViolationsPerEnforcementAction[ea]++
 		logViolation(am.log, r.Constraint, r.EnforcementAction, result)
+		if *emitAuditEvents {
+			emitEvent(r.Constraint, r.EnforcementAction, result, am.eventRecorder, timestamp, am.gkNamespace)
+		}
 	}
 	// log constraints with violations
 	for link := range updateLists {
@@ -626,3 +645,31 @@ func logViolation(l logr.Logger, constraint *unstructured.Unstructured, enforcem
 		logging.ResourceName, violation.rname,
 	)
 }
+
+func emitEvent(constraint *unstructured.Unstructured, enforcementAction string, violation auditResult, eventRecorder record.EventRecorder, timestamp string, gkNamespace string) {
+	annotations := map[string]string{
+		"process":                   "audit",
+		"auditTimestamp":            timestamp,
+		logging.EventType:           "violation_audited",
+		logging.ConstraintKind:      constraint.GetKind(),
+		logging.ConstraintName:      constraint.GetName(),
+		logging.ConstraintNamespace: constraint.GetNamespace(),
+		logging.ConstraintAction:    enforcementAction,
+		logging.ResourceKind:        violation.rkind,
+		logging.ResourceNamespace:   violation.rnamespace,
+		logging.ResourceName:        violation.rname,
+	}
+	reason := "AuditViolation"
+	ref := getViolationRef(gkNamespace, violation.rkind, violation.rname, violation.rnamespace, constraint.GetKind(), constraint.GetName(), constraint.GetNamespace())
+
+	eventRecorder.AnnotatedEventf(ref, annotations, corev1.EventTypeWarning, reason, "Timestamp: %s, Resource Namespace: %s, Constraint: %s, Message: %s", timestamp, violation.rnamespace, constraint.GetName(), violation.message)
+}
+
+func getViolationRef(gkNamespace, rkind, rname, rnamespace, ckind, cname, cnamespace string) *corev1.ObjectReference {
+	return &corev1.ObjectReference{
+		Kind:      rkind,
+		Name:      rname,
+		UID:       types.UID(rkind + "/" + rnamespace + "/" + rname + "/" + ckind + "/" + cnamespace + "/" + cname),
+		Namespace: gkNamespace,
+	}
+}
diff --git a/pkg/webhook/policy.go b/pkg/webhook/policy.go
@@ -34,7 +34,6 @@ import (
 	"github.com/open-policy-agent/gatekeeper/pkg/target"
 	"github.com/open-policy-agent/gatekeeper/pkg/util"
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
-	appsv1 "k8s.io/api/apps/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -47,7 +46,6 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 	clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/client-go/tools/reference"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -66,7 +64,6 @@ var log = logf.Log.WithName("webhook")
 
 const (
 	serviceAccountName = "gatekeeper-admin"
-	deploymentName     = "gatekeeper-controller-manager"
 )
 
 var (
@@ -75,7 +72,7 @@ var (
 	deserializer                       = codecs.UniversalDeserializer()
 	disableEnforcementActionValidation = flag.Bool("disable-enforcementaction-validation", false, "disable validation of the enforcementAction field of a constraint")
 	logDenies                          = flag.Bool("log-denies", false, "log detailed info on each deny")
-	emitDenyEvents                     = flag.Bool("emit-deny-events", false, "emit Kubernetes events in gatekeeper namespace with detailed info on each deny")
+	emitAdmissionEvents                = flag.Bool("emit-admission-events", false, "(alpha) emit Kubernetes events in gatekeeper namespace for each admission violation")
 	serviceaccount                     = fmt.Sprintf("system:serviceaccount:%s:%s", util.GetNamespace(), serviceAccountName)
 	// webhookName is deprecated, set this on the manifest YAML if needed"
 )
@@ -90,38 +87,20 @@ func AddPolicyWebhook(mgr manager.Manager, opa *opa.Client, processExcluder *pro
 		return err
 	}
 	eventBroadcaster := record.NewBroadcaster()
-
 	kubeClient := kubernetes.NewForConfigOrDie(mgr.GetConfig())
-	if err != nil {
-		return err
-	}
 	eventBroadcaster.StartRecordingToSink(&clientcorev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
 	recorder := eventBroadcaster.NewRecorder(
 		scheme.Scheme,
 		corev1.EventSource{Component: "gatekeeper-webhook"})
-	client := mgr.GetClient()
-	var ref *corev1.ObjectReference
-	if *emitDenyEvents {
-		gkDeploy := &appsv1.Deployment{}
-		deploy := types.NamespacedName{Namespace: util.GetNamespace(), Name: deploymentName}
-		err = client.Get(context.Background(), deploy, gkDeploy)
-		if err != nil {
-			return err
-		}
-		ref, err = reference.GetReference(scheme.Scheme, gkDeploy)
-		if err != nil {
-			return err
-		}
-	}
 	wh := &admission.Webhook{
 		Handler: &validationHandler{
 			opa:             opa,
-			client:          client,
+			client:          mgr.GetClient(),
 			reader:          mgr.GetAPIReader(),
 			reporter:        reporter,
 			processExcluder: processExcluder,
 			eventRecorder:   recorder,
-			reference:       ref,
+			gkNamespace:     util.GetNamespace(),
 		},
 	}
 	// TODO(https://github.com/open-policy-agent/gatekeeper/issues/661): remove log injection if the race condition in the cited bug is eliminated.
@@ -146,7 +125,7 @@ type validationHandler struct {
 	injectedConfig  *v1alpha1.Config
 	processExcluder *process.Excluder
 	eventRecorder   record.EventRecorder
-	reference       *corev1.ObjectReference
+	gkNamespace     string
 }
 
 type requestResponse string
@@ -245,7 +224,7 @@ func (h *validationHandler) Handle(ctx context.Context, req admission.Request) a
 func (h *validationHandler) getDenyMessages(res []*rtypes.Result, req admission.Request) []string {
 	var msgs []string
 	var resourceName string
-	if len(res) > 0 && (*logDenies || *emitDenyEvents) {
+	if len(res) > 0 && (*logDenies || *emitAdmissionEvents) {
 		resourceName = req.AdmissionRequest.Name
 		if len(resourceName) == 0 && req.AdmissionRequest.Object.Raw != nil {
 			// On a CREATE operation, the client may omit name and
@@ -271,7 +250,7 @@ func (h *validationHandler) getDenyMessages(res []*rtypes.Result, req admission.
 					"request_username", req.AdmissionRequest.UserInfo.Username,
 				).Info("denied admission")
 			}
-			if *emitDenyEvents && h.reference != nil {
+			if *emitAdmissionEvents {
 				annotations := map[string]string{
 					"process":            "admission",
 					"event_type":         "violation",
@@ -289,7 +268,8 @@ func (h *validationHandler) getDenyMessages(res []*rtypes.Result, req admission.
 					eventMsg = "Dryrun violation"
 					reason = "DryrunViolation"
 				}
-				h.eventRecorder.AnnotatedEventf(h.reference, annotations, corev1.EventTypeWarning, reason, "%s: [Namespace: %s Kind: %s Name: %s] by constraint %s %s", eventMsg, req.AdmissionRequest.Namespace, req.AdmissionRequest.Kind.Kind, resourceName, r.Constraint.GetName(), r.Msg)
+				ref := getViolationRef(h.gkNamespace, req.AdmissionRequest.Kind.Kind, resourceName, req.AdmissionRequest.Namespace, r.Constraint.GetKind(), r.Constraint.GetName(), r.Constraint.GetNamespace())
+				h.eventRecorder.AnnotatedEventf(ref, annotations, corev1.EventTypeWarning, reason, "%s, Resource Namespace: %s, Constraint: %s, Message: %s", eventMsg, req.AdmissionRequest.Namespace, r.Constraint.GetName(), r.Msg)
 			}
 
 		}
@@ -430,3 +410,12 @@ func (h *validationHandler) tracingLevel(ctx context.Context, req admission.Requ
 func (h *validationHandler) skipExcludedNamespace(namespace string) bool {
 	return h.processExcluder.IsNamespaceExcluded(process.Webhook, namespace)
 }
+
+func getViolationRef(gkNamespace, rkind, rname, rnamespace, ckind, cname, cnamespace string) *corev1.ObjectReference {
+	return &corev1.ObjectReference{
+		Kind:      rkind,
+		Name:      rname,
+		UID:       types.UID(rkind + "/" + rnamespace + "/" + rname + "/" + ckind + "/" + cnamespace + "/" + cname),
+		Namespace: gkNamespace,
+	}
+}