Render trace results once (open-policy-agent#393)

* Passing tests

Signed-off-by: John Reese <john@reese.dev>

* Remove table tracing

Signed-off-by: John Reese <john@reese.dev>

acceptance.bats

@@ -76,12 +76,6 @@
   [[ "$output" =~ "data.kubernetes.is_service" ]]
 }
 
-@test "Test command with table output and trace flag" {
-  run ./conftest test -p examples/kubernetes/policy examples/kubernetes/service.yaml -o table --trace
-  [ "$status" -eq 0 ]
-  [[ "$output" =~ "| trace   | examples/kubernetes/service.yaml | Enter data.main.deny = _       |" ]]
-}
-
 @test "Test command with all namespaces flag" {
   run ./conftest test -p examples/docker/policy examples/docker/Dockerfile --all-namespaces
   [ "$status" -eq 1 ]

internal/runner/verify.go

@@ -3,7 +3,6 @@ package runner
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"strings"
 
@@ -46,22 +45,32 @@ func (r *VerifyRunner) Run(ctx context.Context) ([]output.CheckResult, error) {
 
 		buf := new(bytes.Buffer)
 		topdown.PrettyTrace(buf, result.Trace)
-		var traces []error
+		var traces []string
 		for _, line := range strings.Split(buf.String(), "\n") {
 			if len(line) > 0 {
-				traces = append(traces, errors.New(line))
+				traces = append(traces, line)
 			}
 		}
 
+		var outputResult output.Result
+		if result.Fail {
+			outputResult.Message = result.Package + "." + result.Name
+		}
+
+		queryResult := output.QueryResult{
+			Query:   result.Name,
+			Results: []output.Result{outputResult},
+			Traces:  traces,
+		}
+
 		checkResult := output.CheckResult{
 			FileName: result.Location.File,
+			Queries:  []output.QueryResult{queryResult},
 		}
-
-		resultMessage := []output.Result{output.NewResult(result.Package+"."+result.Name, traces)}
 		if result.Fail {
-			checkResult.Failures = resultMessage
+			checkResult.Failures = []output.Result{outputResult}
 		} else {
-			checkResult.Successes = resultMessage
+			checkResult.Successes++
 		}
 
 		results = append(results, checkResult)

output/json.go

@@ -7,23 +7,10 @@ import (
 	"os"
 )
 
-type jsonResult struct {
-	Message  string                 `json:"msg"`
-	Metadata map[string]interface{} `json:"metadata,omitempty"`
-	Traces   []string               `json:"traces,omitempty"`
-}
-
-type jsonCheckResult struct {
-	Filename  string       `json:"filename"`
-	Successes int          `json:"successes"`
-	Warnings  []jsonResult `json:"warnings"`
-	Failures  []jsonResult `json:"failures"`
-}
-
 // JSONOutputManager formats its output to JSON.
 type JSONOutputManager struct {
 	logger  *log.Logger
-	data    []jsonCheckResult
+	data    []CheckResult
 	tracing bool
 }
 
@@ -47,46 +34,15 @@ func (j *JSONOutputManager) WithTracing() OutputManager {
 
 // Put puts the result of the check to the manager in the managers buffer.
 func (j *JSONOutputManager) Put(cr CheckResult) error {
-	if cr.FileName == "-" {
-		cr.FileName = ""
+	if !j.tracing {
+		cr.Queries = nil
 	}
 
-	result := jsonCheckResult{
-		Filename:  cr.FileName,
-		Successes: 0,
-		Warnings:  []jsonResult{},
-		Failures:  []jsonResult{},
-	}
-
-	for _, warning := range cr.Warnings {
-		jsonResult := jsonResult{
-			Message:  warning.Message,
-			Metadata: warning.Metadata,
-		}
-
-		if j.tracing {
-			jsonResult.Traces = errsToStrings(warning.Traces)
-		}
-
-		result.Warnings = append(result.Warnings, jsonResult)
-	}
-
-	for _, failure := range cr.Failures {
-		jsonResult := jsonResult{
-			Message:  failure.Message,
-			Metadata: failure.Metadata,
-		}
-
-		if j.tracing {
-			jsonResult.Traces = errsToStrings(failure.Traces)
-		}
-
-		result.Failures = append(result.Failures, jsonResult)
+	if cr.FileName == "-" {
+		cr.FileName = ""
 	}
 
-	result.Successes = len(cr.Successes)
-	j.data = append(j.data, result)
-
+	j.data = append(j.data, cr)
 	return nil
 }
 
@@ -106,12 +62,3 @@ func (j *JSONOutputManager) Flush() error {
 	j.logger.Print(out.String())
 	return nil
 }
-
-func errsToStrings(errs []error) []string {
-	res := []string{}
-	for _, err := range errs {
-		res = append(res, err.Error())
-	}
-
-	return res
-}

output/json_test.go

@@ -7,41 +7,34 @@ import (
 )
 
 func TestJSON(t *testing.T) {
-	type args struct {
-		fileName string
-		crs      []CheckResult
-	}
-
 	tests := []struct {
-		msg  string
-		args args
-		exp  string
+		name     string
+		input    []CheckResult
+		expected string
 	}{
 		{
-			msg: "no Warnings or errors",
-			args: args{
-				crs: []CheckResult{{FileName: "examples/kubernetes/service.yaml"}},
+			name: "no warnings or errors",
+			input: []CheckResult{
+				{FileName: "examples/kubernetes/service.yaml"},
 			},
-			exp: `[
+			expected: `[
 	{
 		"filename": "examples/kubernetes/service.yaml",
-		"successes": 0,
-		"warnings": [],
-		"failures": []
+		"successes": 0
 	}
 ]
 `,
 		},
 		{
-			msg: "records failure and Warnings",
-			args: args{
-				crs: []CheckResult{{
+			name: "records failures and warnings",
+			input: []CheckResult{
+				{
 					FileName: "examples/kubernetes/service.yaml",
-					Warnings: []Result{NewResult("first warning", []error{})},
-					Failures: []Result{NewResult("first failure", []error{})},
-				}},
+					Warnings: []Result{{Message: "first warning"}},
+					Failures: []Result{{Message: "first failure"}},
+				},
 			},
-			exp: `[
+			expected: `[
 	{
 		"filename": "examples/kubernetes/service.yaml",
 		"successes": 0,
@@ -60,18 +53,17 @@ func TestJSON(t *testing.T) {
 `,
 		},
 		{
-			msg: "mixed failure and Warnings",
-			args: args{
-				crs: []CheckResult{{
+			name: "mixed failure and Warnings",
+			input: []CheckResult{
+				{
 					FileName: "examples/kubernetes/service.yaml",
-					Failures: []Result{NewResult("first failure", []error{})},
-				}},
+					Failures: []Result{{Message: "first failure"}},
+				},
 			},
-			exp: `[
+			expected: `[
 	{
 		"filename": "examples/kubernetes/service.yaml",
 		"successes": 0,
-		"warnings": [],
 		"failures": [
 			{
 				"msg": "first failure"
@@ -82,18 +74,17 @@ func TestJSON(t *testing.T) {
 `,
 		},
 		{
-			msg: "handles stdin input",
-			args: args{
-				fileName: "-",
-				crs: []CheckResult{{
-					Failures: []Result{NewResult("first failure", []error{})}},
+			name: "handles stdin input",
+			input: []CheckResult{
+				{
+					FileName: "-",
+					Failures: []Result{{Message: "first failure"}},
 				},
 			},
-			exp: `[
+			expected: `[
 	{
 		"filename": "",
 		"successes": 0,
-		"warnings": [],
 		"failures": [
 			{
 				"msg": "first failure"
@@ -104,36 +95,31 @@ func TestJSON(t *testing.T) {
 `,
 		},
 		{
-			msg: "multiple check results",
-			args: args{
-				crs: []CheckResult{
-					{FileName: "examples/kubernetes/service.yaml"},
-					{FileName: "examples/kubernetes/deployment.yaml"},
-				},
+			name: "multiple check results",
+			input: []CheckResult{
+				{FileName: "examples/kubernetes/service.yaml"},
+				{FileName: "examples/kubernetes/deployment.yaml"},
 			},
-			exp: `[
+			expected: `[
 	{
 		"filename": "examples/kubernetes/service.yaml",
-		"successes": 0,
-		"warnings": [],
-		"failures": []
+		"successes": 0
 	},
 	{
 		"filename": "examples/kubernetes/deployment.yaml",
-		"successes": 0,
-		"warnings": [],
-		"failures": []
+		"successes": 0
 	}
 ]
 `,
 		},
 	}
+
 	for _, tt := range tests {
-		t.Run(tt.msg, func(t *testing.T) {
+		t.Run(tt.name, func(t *testing.T) {
 			buf := new(bytes.Buffer)
 			s := NewJSONOutputManager(log.New(buf, "", 0))
 
-			for _, cr := range tt.args.crs {
+			for _, cr := range tt.input {
 				if err := s.Put(cr); err != nil {
 					t.Fatalf("put output: %v", err)
 				}
@@ -144,9 +130,8 @@ func TestJSON(t *testing.T) {
 			}
 
 			actual := buf.String()
-
-			if tt.exp != actual {
-				t.Errorf("unexpected output. expected %v got %v", tt.exp, actual)
+			if tt.expected != actual {
+				t.Errorf("unexpected output. expected %v got %v", tt.expected, actual)
 			}
 		})
 	}

output/junit.go

@@ -46,9 +46,7 @@ func (j *JUnitOutputManager) Put(cr CheckResult) error {
 		out := []string{
 			r.Message,
 		}
-		for _, err := range r.Traces {
-			out = append(out, err.Error())
-		}
+
 		return out
 	}
 	convert := func(r Result, status parser.Result) *parser.Test {
@@ -69,12 +67,15 @@ func (j *JUnitOutputManager) Put(cr CheckResult) error {
 	for _, result := range cr.Warnings {
 		j.p.Tests = append(j.p.Tests, convert(result, parser.FAIL))
 	}
+
 	for _, result := range cr.Failures {
 		j.p.Tests = append(j.p.Tests, convert(result, parser.FAIL))
 	}
-	for _, result := range cr.Successes {
-		j.p.Tests = append(j.p.Tests, convert(result, parser.PASS))
+
+	for s := 0; s < cr.Successes; s++ {
+		j.p.Tests = append(j.p.Tests, convert(Result{}, parser.PASS))
 	}
+
 	return nil
 }