Internal Structure Refactoring (#36)

* Internal refactoring
    * Unification of PolicyReports and ClusterPolicyReports processing, APIs still stable
    * DEPRECETED `crdVersion`, Policy Reporter handels now both versions by default
    * DEPRECETED `cleanupDebounceTime`, new internal caching replaced the debounce mechanism, debounce still exist with a fixed period to improve stable metric values.

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+# 1.6.0
+* Internal refactoring
+    * Unification of PolicyReports and ClusterPolicyReports processing, APIs still stable
+    * DEPRECETED `crdVersion`, Policy Reporter handels now both versions by default
+    * DEPRECETED `cleanupDebounceTime`, new internal caching replaced the debounce mechanism, debounce still exist with a fixed period to improve stable metric values.
+
 # 1.5.0
 * Support multiple Resources for a single Result
     * Mapping Result with multiple Resources in multiple Results with a single Resource

charts/policy-reporter/Chart.yaml

@@ -5,8 +5,8 @@ description: |
   It creates Prometheus Metrics and can send rule validation events to different targets like Loki, Elasticsearch, Slack or Discord
 
 type: application
-version: 1.5.0
-appVersion: 1.5.0
+version: 1.6.0
+appVersion: 1.6.0
 
 dependencies:
   - name: monitoring

charts/policy-reporter/templates/deployment.yaml

@@ -47,8 +47,6 @@ spec:
           {{- end }}
           args:
             - --config=/app/config.yaml
-            - --crd-version={{ .Values.crdVersion }}
-            - --cleanup-debounce-time={{ .Values.cleanupDebounceTime }}
             {{- if or .Values.api.enabled .Values.ui.enabled }}
             - --apiPort=8080
             {{- end }}

charts/policy-reporter/values.yaml

@@ -1,7 +1,7 @@
 image:
   repository: fjogeleit/policy-reporter
   pullPolicy: IfNotPresent
-  tag: 1.5.0
+  tag: 1.6.0
 
 imagePullSecrets: []
 
@@ -93,14 +93,12 @@ global:
   # Service Port number
   port: 8080
 
-# PolicyReport CRD Version to use
+# DEPRECTED - Can be removed
+# Policy Reporter watches now for both existing versions by default
 crdVersion: v1alpha1
 
-# Dounce Time in seconds for Modify Events after a cleanup event (Report with 0 Results)
-# Used to prevent Policy Reporter from resending existing violations after Kyverno recreates PolicyReports
-# When an existing Report get an Modify Event with 0 results it waits for the defined amount of time
-# for new Report Events and process the latest incomming Event for this Report which should be the complete recreated Report
-# the required amount of time can be different depending on the amount of validated resources and policies
+# DEPRECTED - Can be removed
+# Policy Reporter uses a new internal cache instead
 cleanupDebounceTime: 20
 
 api:

cmd/root.go

@@ -18,7 +18,6 @@ func NewCLI() *cobra.Command {
 	}
 
 	rootCmd.AddCommand(newRunCMD())
-	rootCmd.AddCommand(newSendCMD())
 
 	return rootCmd
 }

cmd/run.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"flag"
-	"log"
 	"net/http"
 
 	"github.com/fjogeleit/policy-reporter/pkg/config"
@@ -26,8 +25,6 @@ func newRunCMD() *cobra.Command {
 				return err
 			}
 
-			log.Printf("[INFO] Configured DebounceTime %d", c.CleanupDebounceTime)
-
 			var k8sConfig *rest.Config
 			if c.Kubeconfig != "" {
 				k8sConfig, err = clientcmd.BuildConfigFromFlags("", c.Kubeconfig)
@@ -42,26 +39,17 @@ func newRunCMD() *cobra.Command {
 
 			resolver := config.NewResolver(c, k8sConfig)
 
-			pClient, err := resolver.PolicyReportClient(ctx)
-			if err != nil {
-				return err
-			}
-			cpClient, err := resolver.ClusterPolicyReportClient(ctx)
-			if err != nil {
-				return err
-			}
-			rClient, err := resolver.PolicyResultClient(ctx)
+			client, err := resolver.PolicyReportClient(ctx)
 			if err != nil {
 				return err
 			}
 
-			cpClient.RegisterCallback(metrics.CreateClusterPolicyReportMetricsCallback())
-			pClient.RegisterCallback(metrics.CreatePolicyReportMetricsCallback())
+			client.RegisterCallback(metrics.CreateMetricsCallback())
 
 			targets := resolver.TargetClients()
 
 			if len(targets) > 0 {
-				rClient.RegisterPolicyResultCallback(func(r report.Result, e bool) {
+				client.RegisterPolicyResultCallback(func(r report.Result, e bool) {
 					for _, t := range targets {
 						go func(target target.Client, result report.Result, preExisted bool) {
 							if preExisted && target.SkipExistingOnStartup() {
@@ -73,7 +61,7 @@ func newRunCMD() *cobra.Command {
 					}
 				})
 
-				rClient.RegisterPolicyResultWatcher(resolver.SkipExistingOnStartup())
+				client.RegisterPolicyResultWatcher(resolver.SkipExistingOnStartup())
 			}
 
 			errorChan := make(chan error)
@@ -82,8 +70,7 @@ func newRunCMD() *cobra.Command {
 				go func() { errorChan <- resolver.APIServer().Start() }()
 			}
 
-			go func() { errorChan <- cpClient.StartWatching() }()
-			go func() { errorChan <- pClient.StartWatching() }()
+			go func() { errorChan <- client.StartWatching() }()
 
 			go func() {
 				http.Handle("/metrics", promhttp.Handler())
@@ -98,14 +85,8 @@ func newRunCMD() *cobra.Command {
 	// For local usage
 	cmd.PersistentFlags().StringP("kubeconfig", "k", "", "absolute path to the kubeconfig file")
 	cmd.PersistentFlags().StringP("config", "c", "", "target configuration file")
-	cmd.PersistentFlags().StringP("crd-version", "v", "v1alpha1", "Policy Reporter CRD Version")
-	cmd.PersistentFlags().IntP("cleanup-debounce-time", "t", 20, "DebounceTime in Seconds after a Report cleanup started.")
 	cmd.PersistentFlags().IntP("apiPort", "a", 0, "http port for the optional rest api")
 
-	cmd.PersistentFlags().String("loki", "", "loki host: http://loki:3100")
-	cmd.PersistentFlags().String("loki-minimum-priority", "", "Minimum Priority to send Results to Loki (info < warning < critical < error)")
-	cmd.PersistentFlags().Bool("loki-skip-existing-on-startup", false, "Skip Results created before PolicyReporter started. Prevent duplicated sending after new deployment")
-
 	flag.Parse()
 
 	return cmd

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/magiconair/properties v1.8.4 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml v1.8.1 // indirect
 	github.com/prometheus/client_golang v1.9.0
 	github.com/prometheus/client_model v0.2.0

go.sum

@@ -339,6 +339,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=

pkg/api/handler.go

@@ -14,7 +14,7 @@ func PolicyReportHandler(s *report.PolicyReportStore) http.HandlerFunc {
 		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
 		w.WriteHeader(http.StatusOK)
 
-		reports := s.List()
+		reports := s.List("PolicyReport")
 		if len(reports) == 0 {
 			fmt.Fprint(w, "[]")
 
@@ -34,21 +34,21 @@ func PolicyReportHandler(s *report.PolicyReportStore) http.HandlerFunc {
 }
 
 // ClusterPolicyReportHandler for the ClusterPolicyReport REST API
-func ClusterPolicyReportHandler(s *report.ClusterPolicyReportStore) http.HandlerFunc {
+func ClusterPolicyReportHandler(s *report.PolicyReportStore) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
 		w.WriteHeader(http.StatusOK)
 
-		reports := s.List()
+		reports := s.List(report.ClusterPolicyReportType)
 		if len(reports) == 0 {
 			fmt.Fprint(w, "[]")
 
 			return
 		}
 
-		apiReports := make([]ClusterPolicyReport, 0, len(reports))
+		apiReports := make([]PolicyReport, 0, len(reports))
 		for _, r := range reports {
-			apiReports = append(apiReports, mapClusterPolicyReport(r))
+			apiReports = append(apiReports, mapPolicyReport(r))
 		}
 
 		if err := json.NewEncoder(w).Encode(apiReports); err != nil {

pkg/api/handler_test.go

@@ -135,7 +135,7 @@ func Test_ClusterPolicyReportAPI(t *testing.T) {
 		}
 
 		rr := httptest.NewRecorder()
-		handler := http.HandlerFunc(api.ClusterPolicyReportHandler(report.NewClusterPolicyReportStore()))
+		handler := http.HandlerFunc(api.ClusterPolicyReportHandler(report.NewPolicyReportStore()))
 
 		handler.ServeHTTP(rr, req)
 
@@ -170,14 +170,14 @@ func Test_ClusterPolicyReportAPI(t *testing.T) {
 			},
 		}
 
-		creport := report.ClusterPolicyReport{
+		creport := report.PolicyReport{
 			Name:              "cpolr-test",
 			Summary:           report.Summary{},
 			CreationTimestamp: time.Now(),
 			Results:           map[string]report.Result{"": result},
 		}
 
-		store := report.NewClusterPolicyReportStore()
+		store := report.NewPolicyReportStore()
 		store.Add(creport)
 
 		rr := httptest.NewRecorder()

pkg/api/model.go

@@ -41,15 +41,7 @@ type Summary struct {
 // PolicyReport API Model
 type PolicyReport struct {
 	Name              string    `json:"name"`
-	Namespace         string    `json:"namespace"`
-	Results           []Result  `json:"results"`
-	Summary           Summary   `json:"summary"`
-	CreationTimestamp time.Time `json:"creationTimestamp"`
-}
-
-// ClusterPolicyReport API Model
-type ClusterPolicyReport struct {
-	Name              string    `json:"name"`
+	Namespace         string    `json:"namespace,omitempty"`
 	Results           []Result  `json:"results"`
 	Summary           Summary   `json:"summary"`
 	CreationTimestamp time.Time `json:"creationTimestamp"`
@@ -98,48 +90,6 @@ func mapPolicyReport(p report.PolicyReport) PolicyReport {
 	}
 }
 
-func mapClusterPolicyReport(c report.ClusterPolicyReport) ClusterPolicyReport {
-	results := make([]Result, 0, len(c.Results))
-
-	for _, r := range c.Results {
-		result := Result{
-			Message:  r.Message,
-			Policy:   r.Policy,
-			Rule:     r.Rule,
-			Priority: r.Priority.String(),
-			Status:   r.Status,
-			Severity: r.Severity,
-			Category: r.Category,
-			Scored:   r.Scored,
-		}
-
-		if r.HasResource() {
-			result.Resource = &Resource{
-				Namespace:  r.Resource.Namespace,
-				APIVersion: r.Resource.APIVersion,
-				Kind:       r.Resource.Kind,
-				Name:       r.Resource.Name,
-				UID:        r.Resource.UID,
-			}
-		}
-
-		results = append(results, result)
-	}
-
-	return ClusterPolicyReport{
-		Name:              c.Name,
-		CreationTimestamp: c.CreationTimestamp,
-		Summary: Summary{
-			Skip:  c.Summary.Skip,
-			Pass:  c.Summary.Pass,
-			Warn:  c.Summary.Warn,
-			Fail:  c.Summary.Fail,
-			Error: c.Summary.Error,
-		},
-		Results: results,
-	}
-}
-
 // Target API Model
 type Target struct {
 	Name                  string `json:"name"`