CEL test coverage calculation (#1209)

* added cel test coverage calculation and tests

Author: aakash070

policy/BUILD.bazel

@@ -91,6 +91,7 @@ cel_go_test(
     name = "context_pb_policy",
     cel_expr = "testdata/context_pb/policy.yaml",
     config = "testdata/context_pb/config.yaml",
+    enable_coverage = True,
     file_descriptor_set = ":test_all_types_fds",
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/context_pb/tests.yaml",
@@ -101,6 +102,7 @@ cel_go_test(
     cel_expr = "testdata/k8s/policy.yaml",
     config = "testdata/k8s/config.yaml",
     deps = ["//policy:go_default_library"],
+    enable_coverage = True,
     test_src = "//policy/test:k8s_cel_test_runner.go",
     test_suite = "testdata/k8s/tests.yaml",
 )
@@ -109,6 +111,7 @@ cel_go_test(
     name = "limits_policy",
     cel_expr = "testdata/limits/policy.yaml",
     config = "testdata/limits/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/limits/tests.yaml",
 )
@@ -117,6 +120,7 @@ cel_go_test(
     name = "nested_rule_policy",
     cel_expr = "testdata/nested_rule/policy.yaml",
     config = "testdata/nested_rule/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/nested_rule/tests.yaml",
 )
@@ -125,6 +129,7 @@ cel_go_test(
     name = "nested_rule2_policy",
     cel_expr = "testdata/nested_rule2/policy.yaml",
     config = "testdata/nested_rule2/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/nested_rule2/tests.yaml",
 )
@@ -133,6 +138,7 @@ cel_go_test(
     name = "nested_rule3_policy",
     cel_expr = "testdata/nested_rule3/policy.yaml",
     config = "testdata/nested_rule3/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/nested_rule3/tests.yaml",
 )
@@ -141,6 +147,7 @@ cel_go_test(
     name = "nested_rule4_policy",
     cel_expr = "testdata/nested_rule4/policy.yaml",
     config = "testdata/nested_rule4/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/nested_rule4/tests.yaml",
 )
@@ -149,6 +156,7 @@ cel_go_test(
     name = "nested_rule5_policy",
     cel_expr = "testdata/nested_rule5/policy.yaml",
     config = "testdata/nested_rule5/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/nested_rule5/tests.yaml",
 )
@@ -157,6 +165,7 @@ cel_go_test(
     name = "nested_rule6_policy",
     cel_expr = "testdata/nested_rule6/policy.yaml",
     config = "testdata/nested_rule6/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/nested_rule6/tests.yaml",
 )
@@ -165,6 +174,7 @@ cel_go_test(
     name = "nested_rule7_policy",
     cel_expr = "testdata/nested_rule7/policy.yaml",
     config = "testdata/nested_rule7/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/nested_rule7/tests.yaml",
 )
@@ -174,6 +184,7 @@ cel_go_test(
     cel_expr = "testdata/pb/policy.yaml",
     config = "testdata/pb/config.yaml",
     file_descriptor_set = ":test_all_types_fds",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/pb/tests.yaml",
 )
@@ -182,6 +193,7 @@ cel_go_test(
     name = "required_labels_policy",
     cel_expr = "testdata/required_labels/policy.yaml",
     config = "testdata/required_labels/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/required_labels/tests.yaml",
 )
@@ -190,6 +202,7 @@ cel_go_test(
     name = "restricted_destinations_policy",
     cel_expr = "testdata/restricted_destinations/policy.yaml",
     config = "testdata/restricted_destinations/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/restricted_destinations/tests.yaml",
 )
@@ -198,6 +211,7 @@ cel_go_test(
     name = "unnest_policy",
     cel_expr = "testdata/unnest/policy.yaml",
     config = "testdata/unnest/config.yaml",
+    enable_coverage = True,
     test_src = "//policy/test:cel_test_runner.go",
     test_suite = "testdata/unnest/tests.yaml",
 )

test/cel_go_test.bzl

@@ -31,6 +31,7 @@ def cel_go_test(
         test_data_path = "",
         file_descriptor_set = "",
         filegroup = "",
+        enable_coverage = False,
         deps = [],
         data = [],
         **kwargs):
@@ -59,6 +60,7 @@ def cel_go_test(
           this must be in binary format with either a .binarypb or .pb or.fds extension. If you need
           to support a textformat file_descriptor_set, embed it in the environment file. (default None)
       filegroup: str label of a filegroup containing the test suite, config, and cel_expr.
+      enable_coverage: boolean indicating if coverage should be enabled for the test.
       deps: list of dependencies for the go_test rule
       data: list of data dependencies for the go_test rule
       **kwargs: additional arguments to pass to the go_test rule
@@ -104,6 +106,7 @@ def cel_go_test(
         "--test_suite_path=%s" % test_suite,
         "--config_path=%s" % config,
         "--base_config_path=%s" % base_config,
+        "--enable_coverage=%s" % enable_coverage,
     ]
 
     if cel_expr_format == ".cel" or cel_expr_format == ".celpolicy" or cel_expr_format == ".yaml":

tools/celtest/BUILD.bazel

@@ -23,14 +23,18 @@ package(
 go_library(
     name = "go_default_library",
     srcs = [
+        "test_coverage_reporter.go",
         "test_runner.go",
     ],
     importpath = "github.com/google/cel-go/tools/celtest",
     deps = [
         "//cel:go_default_library",
+        "//common/ast:go_default_library",
+        "//common/operators:go_default_library",
         "//common/types:go_default_library",
         "//common/types/ref:go_default_library",
         "//interpreter:go_default_library",
+        "//parser:go_default_library",
         "//test:go_default_library",
         "//tools/compiler:go_default_library",
         "@com_github_google_go_cmp//cmp:go_default_library",
@@ -54,6 +58,7 @@ go_test(
     name = "go_default_test",
     size = "small",
     srcs = [
+        "test_coverage_reporter_test.go",
         "test_runner_test.go",
     ],
     data = [
@@ -63,6 +68,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//cel:go_default_library",
+        "//common/ast:go_default_library",
         "//common/decls:go_default_library",
         "//common/types:go_default_library",
         "//common/types/ref:go_default_library",
@@ -84,6 +90,7 @@ cel_go_test(
     name = "pb_policy",
     cel_expr = "testdata/pb/policy.yaml",
     config = "testdata/pb/config.yaml",
+    enable_coverage = True,
     file_descriptor_set = "//policy:test_all_types_fds",
     test_data_path = "//policy",
     test_src = "//policy/test:cel_test_runner.go",

tools/celtest/test_coverage_reporter.go

@@ -0,0 +1,152 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package celtest provides functions for testing CEL policies and expressions.
+package celtest
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/cel-go/common/ast"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/parser"
+)
+
+// reportCoverage reports the coverage information for the provided programs.
+//   - For the node coverage, the coverage is reported as a percentage of the number of nodes which
+//     were evaluated during the test execution and hence are present in the program Coverage report.
+//   - For the branch coverage, every node which has a boolean return type is considered as a branch.
+//     The number of branches which were evaluated during the test execution and hence are present in
+//     the program Coverage report are reported as the branch coverage percentage.
+func reportCoverage(t *testing.T, programs []Program) {
+	t.Helper()
+	for _, p := range programs {
+		a := p.Ast.NativeRep()
+		exprString, err := parser.Unparse(a.Expr(), a.SourceInfo(), parser.WrapOnColumn(70))
+		if err != nil {
+			t.Logf("Error converting AST to string for a program: %v", err)
+			continue
+		}
+		rootNavigableExpr := ast.NavigateAST(p.Ast.NativeRep())
+		// Initialize coverage metrics
+		cr := &coverageReport{
+			nodes:                  0,
+			coveredNodes:           0,
+			branches:               0,
+			coveredBooleanOutcomes: 0,
+			unencounteredNodes:     []string{},
+			unencounteredBranches:  []string{},
+		}
+		traverseAndCalculateCoverage(t, rootNavigableExpr, p, true, "", cr)
+		printCoverageReport(t, exprString, cr)
+	}
+}
+
+type coverageReport struct {
+	nodes                  int64
+	coveredNodes           int64
+	branches               int64
+	coveredBooleanOutcomes int64
+	unencounteredNodes     []string
+	unencounteredBranches  []string
+}
+
+func traverseAndCalculateCoverage(t *testing.T, expr ast.NavigableExpr, p Program, logUnencountered bool,
+	preceedingTabs string, cr *coverageReport) {
+	t.Helper()
+	if expr == nil || len(p.CoverageStats) == 0 {
+		return
+	}
+	nodeID := expr.ID()
+	exprText, err := parser.Unparse(expr, p.Ast.NativeRep().SourceInfo(), parser.WrapOnColumn(70))
+	if err != nil {
+		t.Logf("Error converting AST to string for an expression: %v", err)
+		return
+	}
+	cr.nodes++
+	// Check for nodes which need to be logged for missing coverage:
+	// * node should be of boolean type
+	// * node should not a literal
+	// * cel.@block type function nodes are bypassed as they are just the container for
+	// the underlying expressions and the node itself does not offer any significant coverage information
+	interestingBoolNode := expr.Type() == types.BoolType && expr.AsLiteral() == nil && expr.AsCall().FunctionName() != "cel.@block"
+	// Check for node coverage
+	if _, isCovered := p.CoverageStats[nodeID]; isCovered {
+		cr.coveredNodes++
+	} else if logUnencountered {
+		if interestingBoolNode {
+			cr.unencounteredNodes = append(cr.unencounteredNodes,
+				fmt.Sprintf("\nExpression ID %d ('%s')", nodeID, exprText))
+		}
+		logUnencountered = false
+	}
+	// Check for Branch Coverage if the node is a boolean type
+	if interestingBoolNode {
+		cr.branches += 2
+		if info, found := p.CoverageStats[nodeID]; !found {
+			if logUnencountered {
+				cr.unencounteredBranches = append(cr.unencounteredBranches,
+					"\n"+preceedingTabs+fmt.Sprintf("Expression ID %d ('%s'): No coverage", nodeID, exprText))
+				preceedingTabs = preceedingTabs + "\t\t"
+			}
+		} else {
+			if _, ok := info[types.True]; ok {
+				cr.coveredBooleanOutcomes++
+			} else if logUnencountered {
+				cr.unencounteredBranches = append(cr.unencounteredBranches,
+					"\n"+preceedingTabs+fmt.Sprintf("Expression ID %d ('%s'): lacks 'true' coverage", nodeID, exprText))
+				preceedingTabs = preceedingTabs + "\t\t"
+
+			}
+			if _, ok := info[types.False]; ok {
+				cr.coveredBooleanOutcomes++
+			} else if logUnencountered {
+				cr.unencounteredBranches = append(cr.unencounteredBranches,
+					"\n"+preceedingTabs+fmt.Sprintf("Expression ID %d ('%s'): lacks 'false' coverage", nodeID, exprText))
+				preceedingTabs = preceedingTabs + "\t\t"
+			}
+		}
+	}
+	for _, child := range expr.Children() {
+		traverseAndCalculateCoverage(t, child.(ast.NavigableExpr), p, logUnencountered, preceedingTabs, cr)
+	}
+}
+
+func printCoverageReport(t *testing.T, exprString string, cr *coverageReport) {
+	t.Helper()
+	t.Logf("--- Start Coverage Report ---\nExpression: %s", exprString)
+	if cr.nodes == 0 {
+		t.Logf("No coverage stats found")
+		return
+	}
+	// Log Node Coverage results
+	nodeCoverage := float64(cr.coveredNodes) / float64(cr.nodes) * 100.0
+	t.Logf("AST Node Coverage: %.2f%% (%d out of %d nodes covered)", nodeCoverage, cr.coveredNodes, cr.nodes)
+	if len(cr.unencounteredNodes) > 0 {
+		t.Logf("Interesting Unencountered Nodes:\n%s", strings.Join(cr.unencounteredNodes, "\n"))
+	}
+	// Log Branch Coverage results
+	branchCoverage := 0.0
+	if cr.branches > 0 {
+		branchCoverage = float64(cr.coveredBooleanOutcomes) / float64(cr.branches) * 100.0
+	}
+	t.Logf("AST Branch Coverage: %.2f%% (%d out of %d branch outcomes covered)", branchCoverage,
+		cr.coveredBooleanOutcomes, cr.branches)
+	if len(cr.unencounteredBranches) > 0 {
+		t.Logf("Interesting Unencountered Branch Paths:\n%s", strings.Join(cr.unencounteredBranches, "\n"))
+	}
+	t.Logf("--- End Coverage Report ---\n")
+}