fix: cover built-in errors with a flag, suppress unless it's passed (#…

…871)

Signed-off-by: boranx <boran.seref@gmail.com>

acceptance.bats

@@ -505,7 +505,7 @@ EOF"
 }
 
 @test "Should fail evaluation if a builtin function returns error" {
-  run ./conftest test -p examples/builtin-errors/invalid-dns.rego examples/kubernetes/deployment.yaml
+  run ./conftest test --show-builtin-errors -p examples/builtin-errors/invalid-dns.rego examples/kubernetes/deployment.yaml
   [ "$status" -eq 1 ]
   [[ "$output" =~ "built-in error" ]]
 }

internal/commands/test.go

@@ -100,6 +100,7 @@ func NewTestCommand(ctx context.Context) *cobra.Command {
 				"capabilities",
 				"trace",
 				"strict",
+				"show-builtin-errors",
 				"update",
 				"junit-hide-message",
 				"quiet",
@@ -168,6 +169,7 @@ func NewTestCommand(ctx context.Context) *cobra.Command {
 
 	cmd.Flags().Bool("trace", false, "Enable more verbose trace output for Rego queries")
 	cmd.Flags().Bool("strict", false, "Enable strict mode for Rego policies")
+	cmd.Flags().Bool("show-builtin-errors", false, "Collect and return all encountered built-in errors")
 	cmd.Flags().Bool("combine", false, "Combine all config files to be evaluated together")
 
 	cmd.Flags().String("ignore", "", "A regex pattern which can be used for ignoring paths")

policy/engine.go

@@ -25,12 +25,13 @@ import (
 
 // Engine represents the policy engine.
 type Engine struct {
-	trace    bool
-	modules  map[string]*ast.Module
-	compiler *ast.Compiler
-	store    storage.Store
-	policies map[string]string
-	docs     map[string]string
+	trace         bool
+	builtinErrors bool
+	modules       map[string]*ast.Module
+	compiler      *ast.Compiler
+	store         storage.Store
+	policies      map[string]string
+	docs          map[string]string
 }
 
 type compilerOptions struct {
@@ -156,6 +157,10 @@ func (e *Engine) EnableTracing() {
 	e.trace = true
 }
 
+func (e *Engine) ShowBuiltinErrors() {
+	e.builtinErrors = true
+}
+
 // Check executes all of the loaded policies against the input and returns the results.
 func (e *Engine) Check(ctx context.Context, configs map[string]interface{}, namespace string) ([]output.CheckResult, error) {
 	var checkResults []output.CheckResult
@@ -446,7 +451,7 @@ func (e *Engine) query(ctx context.Context, input interface{}, query string) (ou
 		return output.QueryResult{}, fmt.Errorf("evaluating policy: %w", err)
 	}
 
-	if len(*builtInErrors) > 0 {
+	if e.builtinErrors && len(*builtInErrors) > 0 {
 		return output.QueryResult{}, fmt.Errorf("built-in error: %+v", (*builtInErrors))
 	}
 

runner/test.go

@@ -30,6 +30,7 @@ type TestRunner struct {
 	NoColor            bool `mapstructure:"no-color"`
 	NoFail             bool `mapstructure:"no-fail"`
 	SuppressExceptions bool `mapstructure:"suppress-exceptions"`
+	ShowBuiltinErrors  bool `mapstructure:"show-builtin-errors"`
 	Combine            bool
 	Quiet              bool
 	Output             string
@@ -70,6 +71,10 @@ func (t *TestRunner) Run(ctx context.Context, fileList []string) ([]output.Check
 		engine.EnableTracing()
 	}
 
+	if t.ShowBuiltinErrors {
+		engine.ShowBuiltinErrors()
+	}
+
 	namespaces := t.Namespace
 	if t.AllNamespaces {
 		namespaces = engine.Namespaces()