From cb2c95ac941471ed6908459beed10f6617d0c2e5 Mon Sep 17 00:00:00 2001 From: Divya Rani Date: Thu, 6 Aug 2020 00:38:38 +0530 Subject: [PATCH 1/3] sort benchmarks in scorecard --- .../kubectl-mtb/internal/kubectl-mtb/run.go | 22 ++++++++++-------- .../internal/reporter/default_reporter.go | 23 ++++++++----------- .../kubectl-mtb/pkg/benchmark/benchmark.go | 1 + .../pkg/benchmark_suite/benchmarks_suite.go | 7 ++++-- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go b/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go index d08416715..1d91b4530 100644 --- a/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go +++ b/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go @@ -109,18 +109,18 @@ func reportSuiteDidEnd(suiteSummary *reporter.SuiteSummary, reportersArray []rep func removeBenchmarksWithIDs(ids []string) { temp := []*benchmark.Benchmark{} - for _, benchmark := range benchmarks { - found := false - for _, id := range ids { - if(benchmark.ID == id) { - found = true - } + for _, benchmark := range benchmarks { + found := false + for _, id := range ids { + if benchmark.ID == id { + found = true } + } - if !found { - temp = append(temp, benchmark) - } + if !found { + temp = append(temp, benchmark) } + } benchmarks = temp } @@ -164,7 +164,6 @@ func runTests(cmd *cobra.Command, args []string) error { skipIDs := strings.Split(skipFlag, ",") removeBenchmarksWithIDs(skipIDs) - suiteSummary := &reporter.SuiteSummary{ Suite: test.BenchmarkSuite, NumberOfTotalTests: len(benchmarks), @@ -193,6 +192,7 @@ func runTests(cmd *cobra.Command, args []string) error { suiteSummary.NumberOfFailedValidations++ ts.Validation = false ts.ValidationError = err + b.Status = "Error" } // Check PreRun status @@ -202,8 +202,10 @@ func runTests(cmd *cobra.Command, args []string) error { suiteSummary.NumberOfFailedTests++ ts.Test = false ts.TestError = err + b.Status = "Fail" } else { suiteSummary.NumberOfPassedTests++ + b.Status = "Pass" } } elapsed := time.Since(startTest) diff --git a/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go b/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go index fd44f316d..29e80d183 100644 --- a/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go +++ b/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go @@ -5,9 +5,9 @@ import ( "os" "strconv" - v1alpha1 "github.com/kubernetes-sigs/wg-policy-prototypes/policy-report/api/v1alpha1" "github.com/olekukonko/tablewriter" "sigs.k8s.io/multi-tenancy/benchmarks/kubectl-mtb/pkg/benchmark" + benchmarksuite "sigs.k8s.io/multi-tenancy/benchmarks/kubectl-mtb/pkg/benchmark_suite" ) // DefaultReporter collects all the test summaries @@ -15,8 +15,6 @@ type DefaultReporter struct { testSummaries []*TestSummary } -var testResult = map[*benchmark.Benchmark]v1alpha1.PolicyStatus{} - // NewDefaultReporter returns the pointer of DefaultReporter func NewDefaultReporter() *DefaultReporter { return &DefaultReporter{} @@ -35,11 +33,9 @@ func (r *DefaultReporter) TestWillRun(testSummary *TestSummary) { writer.Println(0, testSummary.Benchmark.Title) writer.Println(0, writer.Colorize(grayColor, "%s", testSummary.Benchmark.Description)) if testSummary.Test { - testResult[testSummary.Benchmark] = "Pass" passed := "Passed " + tick writer.Println(0, writer.Colorize(greenColor, passed)) } else { - testResult[testSummary.Benchmark] = "Fail" failed := "Failed " + cross writer.Println(0, writer.Colorize(redColor, failed)) writer.Print(0, writer.Colorize(lilac, "Remediation: ")) @@ -49,7 +45,6 @@ func (r *DefaultReporter) TestWillRun(testSummary *TestSummary) { writer.PrintBanner(writer.Colorize(grayColor, "Completed in %v", testSummary.RunTime), "-") return } - testResult[testSummary.Benchmark] = "Error" preRunfmt := writer.Colorize(magentaColor, "[PreRun-Validation Error]") errormsg := writer.Colorize(redColor, testSummary.ValidationError.Error()) bannerText := fmt.Sprintf("%s [%s] %s: %s %s", preRunfmt, testSummary.Benchmark.ID, testSummary.Benchmark.Title, errormsg, cross) @@ -66,31 +61,31 @@ func (r *DefaultReporter) SuiteDidEnd(suiteSummary *SuiteSummary) { writer.PrintNewLine() writer.PrintBanner(writer.Colorize(grayColor, "Completed in %v", suiteSummary.RunTime), "=") - printScoreCard(testResult) + printScoreCard(benchmarksuite.SortedBenchmarks) } // FullSummary prints end result of all the tests at one place. -func printScoreCard(testResult map[*benchmark.Benchmark]v1alpha1.PolicyStatus) { +func printScoreCard(benchmarks []*benchmark.Benchmark) { data := [][]string{} counter := 0 - for val, key := range testResult{ + for _, b := range benchmarks { counter++ var status string - - switch key { + + switch b.Status { case "Error": status = writer.Colorize(magentaColor, "Error") case "Pass": status = writer.Colorize(greenColor, "Passed") case "Fail": status = writer.Colorize(redColor, "Failed") - case "Skip": + default: status = writer.Colorize(yellowColor, "Skipped") } - testName := val.Title - result := []string{strconv.Itoa(counter), val.ID, testName, status} + testName := b.Title + result := []string{strconv.Itoa(counter), b.ID, testName, status} data = append(data, result) } diff --git a/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go b/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go index 281e4bc3b..50ed1939c 100644 --- a/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go +++ b/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go @@ -16,6 +16,7 @@ type Benchmark struct { Description string `yaml:"description"` Remediation string `yaml:"remediation"` ProfileLevel int `yaml:"profileLevel"` + Status string `yaml:"status"` PreRun func(string, *kubernetes.Clientset, *kubernetes.Clientset) error Run func(string, *kubernetes.Clientset, *kubernetes.Clientset) error } diff --git a/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go b/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go index db6dac3dc..01d2819f7 100644 --- a/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go +++ b/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go @@ -15,6 +15,9 @@ type BenchmarkSuite struct { Benchmarks []*benchmark.Benchmark } +// SortedBenchmarks contains benchmarks sorted according to profile level, category and id +var SortedBenchmarks []*benchmark.Benchmark + // Totals returns count of benchmarks in Benchmark Suite func (bs *BenchmarkSuite) Totals() int { return len(bs.Benchmarks) @@ -33,8 +36,8 @@ func (bs *BenchmarkSuite) ProfileLevel(pl int) []*benchmark.Benchmark { benchmarksArray = append(benchmarksArray, b) } } - sortedBenchmarks := sortBenchmarks(benchmarksArray) - return sortedBenchmarks + SortedBenchmarks = sortBenchmarks(benchmarksArray) + return SortedBenchmarks } // sortBenchmarks returns slice of Benchmarks sorted according to Profile level, category and id respectively From 3170ce75ef8baea54ae9e667b904950b78b16ebe Mon Sep 17 00:00:00 2001 From: Divya Rani Date: Thu, 6 Aug 2020 00:38:38 +0530 Subject: [PATCH 2/3] sort benchmarks in scorecard --- .../kubectl-mtb/internal/kubectl-mtb/run.go | 3 +++ .../internal/reporter/default_reporter.go | 23 ++++++++----------- .../pkg/benchmark_suite/benchmarks_suite.go | 7 ++++-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go b/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go index 846be7ac1..5ca5b0251 100644 --- a/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go +++ b/benchmarks/kubectl-mtb/internal/kubectl-mtb/run.go @@ -189,6 +189,7 @@ func runTests(cmd *cobra.Command, args []string) error { suiteSummary.NumberOfFailedValidations++ ts.Validation = false ts.ValidationError = err + b.Status = "Error" } // Check PreRun status @@ -198,8 +199,10 @@ func runTests(cmd *cobra.Command, args []string) error { suiteSummary.NumberOfFailedTests++ ts.Test = false ts.TestError = err + b.Status = "Fail" } else { suiteSummary.NumberOfPassedTests++ + b.Status = "Pass" } } diff --git a/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go b/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go index fd44f316d..29e80d183 100644 --- a/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go +++ b/benchmarks/kubectl-mtb/internal/reporter/default_reporter.go @@ -5,9 +5,9 @@ import ( "os" "strconv" - v1alpha1 "github.com/kubernetes-sigs/wg-policy-prototypes/policy-report/api/v1alpha1" "github.com/olekukonko/tablewriter" "sigs.k8s.io/multi-tenancy/benchmarks/kubectl-mtb/pkg/benchmark" + benchmarksuite "sigs.k8s.io/multi-tenancy/benchmarks/kubectl-mtb/pkg/benchmark_suite" ) // DefaultReporter collects all the test summaries @@ -15,8 +15,6 @@ type DefaultReporter struct { testSummaries []*TestSummary } -var testResult = map[*benchmark.Benchmark]v1alpha1.PolicyStatus{} - // NewDefaultReporter returns the pointer of DefaultReporter func NewDefaultReporter() *DefaultReporter { return &DefaultReporter{} @@ -35,11 +33,9 @@ func (r *DefaultReporter) TestWillRun(testSummary *TestSummary) { writer.Println(0, testSummary.Benchmark.Title) writer.Println(0, writer.Colorize(grayColor, "%s", testSummary.Benchmark.Description)) if testSummary.Test { - testResult[testSummary.Benchmark] = "Pass" passed := "Passed " + tick writer.Println(0, writer.Colorize(greenColor, passed)) } else { - testResult[testSummary.Benchmark] = "Fail" failed := "Failed " + cross writer.Println(0, writer.Colorize(redColor, failed)) writer.Print(0, writer.Colorize(lilac, "Remediation: ")) @@ -49,7 +45,6 @@ func (r *DefaultReporter) TestWillRun(testSummary *TestSummary) { writer.PrintBanner(writer.Colorize(grayColor, "Completed in %v", testSummary.RunTime), "-") return } - testResult[testSummary.Benchmark] = "Error" preRunfmt := writer.Colorize(magentaColor, "[PreRun-Validation Error]") errormsg := writer.Colorize(redColor, testSummary.ValidationError.Error()) bannerText := fmt.Sprintf("%s [%s] %s: %s %s", preRunfmt, testSummary.Benchmark.ID, testSummary.Benchmark.Title, errormsg, cross) @@ -66,31 +61,31 @@ func (r *DefaultReporter) SuiteDidEnd(suiteSummary *SuiteSummary) { writer.PrintNewLine() writer.PrintBanner(writer.Colorize(grayColor, "Completed in %v", suiteSummary.RunTime), "=") - printScoreCard(testResult) + printScoreCard(benchmarksuite.SortedBenchmarks) } // FullSummary prints end result of all the tests at one place. -func printScoreCard(testResult map[*benchmark.Benchmark]v1alpha1.PolicyStatus) { +func printScoreCard(benchmarks []*benchmark.Benchmark) { data := [][]string{} counter := 0 - for val, key := range testResult{ + for _, b := range benchmarks { counter++ var status string - - switch key { + + switch b.Status { case "Error": status = writer.Colorize(magentaColor, "Error") case "Pass": status = writer.Colorize(greenColor, "Passed") case "Fail": status = writer.Colorize(redColor, "Failed") - case "Skip": + default: status = writer.Colorize(yellowColor, "Skipped") } - testName := val.Title - result := []string{strconv.Itoa(counter), val.ID, testName, status} + testName := b.Title + result := []string{strconv.Itoa(counter), b.ID, testName, status} data = append(data, result) } diff --git a/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go b/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go index db6dac3dc..01d2819f7 100644 --- a/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go +++ b/benchmarks/kubectl-mtb/pkg/benchmark_suite/benchmarks_suite.go @@ -15,6 +15,9 @@ type BenchmarkSuite struct { Benchmarks []*benchmark.Benchmark } +// SortedBenchmarks contains benchmarks sorted according to profile level, category and id +var SortedBenchmarks []*benchmark.Benchmark + // Totals returns count of benchmarks in Benchmark Suite func (bs *BenchmarkSuite) Totals() int { return len(bs.Benchmarks) @@ -33,8 +36,8 @@ func (bs *BenchmarkSuite) ProfileLevel(pl int) []*benchmark.Benchmark { benchmarksArray = append(benchmarksArray, b) } } - sortedBenchmarks := sortBenchmarks(benchmarksArray) - return sortedBenchmarks + SortedBenchmarks = sortBenchmarks(benchmarksArray) + return SortedBenchmarks } // sortBenchmarks returns slice of Benchmarks sorted according to Profile level, category and id respectively From 372bfc8fe2830ef0697cc9665712bd7490b00788 Mon Sep 17 00:00:00 2001 From: Divya Rani Date: Fri, 14 Aug 2020 10:07:17 +0530 Subject: [PATCH 3/3] minor fix --- benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go b/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go index 44c2f9a6e..4633c4fca 100644 --- a/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go +++ b/benchmarks/kubectl-mtb/pkg/benchmark/benchmark.go @@ -16,6 +16,7 @@ type Benchmark struct { Description string `yaml:"description"` Remediation string `yaml:"remediation"` ProfileLevel int `yaml:"profileLevel"` + Status string `yaml:"status"` PreRun func(types.RunOptions) error Run func(types.RunOptions) error PostRun func(types.RunOptions) error