From 4010d6eba4be4440570e2485eaac49da26f16235 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 30 Sep 2025 21:44:15 -0700 Subject: [PATCH] Support for Dot graph via graphviz PiperOrigin-RevId: 813572385 --- testing/testrunner/BUILD | 2 + testing/testrunner/coverage_index.cc | 60 ++++++- testing/testrunner/coverage_index.h | 9 ++ testing/testrunner/runner_lib_test.cc | 219 ++++++++++++++++++++++++-- testing/testrunner/user_tests/BUILD | 1 + 5 files changed, 279 insertions(+), 12 deletions(-) diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD index 0e8be6ca9..013f0ed7f 100644 --- a/testing/testrunner/BUILD +++ b/testing/testrunner/BUILD @@ -108,6 +108,7 @@ cc_test( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -163,6 +164,7 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", ], ) diff --git a/testing/testrunner/coverage_index.cc b/testing/testrunner/coverage_index.cc index bdc3febc4..3fb5a9c3b 100644 --- a/testing/testrunner/coverage_index.cc +++ b/testing/testrunner/coverage_index.cc @@ -24,6 +24,9 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" #include "common/ast.h" #include "common/value.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" @@ -44,6 +47,32 @@ using ::google::api::expr::runtime::CelExpressionBuilder; using ::google::api::expr::runtime::Instrumentation; using ::google::api::expr::runtime::InstrumentationFactory; +std::string EscapeSpecialCharacters(absl::string_view expr_text) { + return absl::StrReplaceAll(expr_text, {{"\\\"", "\""}, + {"\"", "\\\""}, + {"\n", "\\n"}, + {"||", " \\| \\| "}, + {"<", "\\<"}, + {">", "\\>"}, + {"{", "\\{"}, + {"}", "\\}"}}); +} + +std::string KindToString(const NavigableProtoAstNode& node) { + if (node.parent_relation() != ChildKind::kUnspecified && + node.parent()->expr()->has_comprehension_expr()) { + const cel::expr::Expr::Comprehension& comp = + node.parent()->expr()->comprehension_expr(); + if (node.expr()->id() == comp.iter_range().id()) return "IterRange"; + if (node.expr()->id() == comp.accu_init().id()) return "AccuInit"; + if (node.expr()->id() == comp.loop_condition().id()) return "LoopCondition"; + if (node.expr()->id() == comp.loop_step().id()) return "LoopStep"; + if (node.expr()->id() == comp.result().id()) return "Result"; + } + + return absl::StrCat(NodeKindName(node.node_kind()), " Node"); +} + const Type* absl_nullable FindCheckerType(const CheckedExpr& expr, int64_t expr_id) { if (auto it = expr.type_map().find(expr_id); it != expr.type_map().end()) { @@ -69,7 +98,7 @@ void TraverseAndCalculateCoverage( const absl::flat_hash_map& stats_map, bool log_unencountered, std::string preceeding_tabs, - CoverageIndex::CoverageReport& report) { + CoverageIndex::CoverageReport& report, std::string& dot_graph) { int64_t node_id = node.expr()->id(); const CoverageIndex::NodeCoverageStats& stats = stats_map.at(node_id); @@ -84,6 +113,24 @@ void TraverseAndCalculateCoverage( (!node.expr()->has_call_expr() || node.expr()->call_expr().function() != "cel.@block"); + absl::string_view node_coverage_style = kUncoveredNodeStyle; + if (stats.covered) { + if (is_interesting_bool_node) { + if (stats.has_true_branch && stats.has_false_branch) { + node_coverage_style = kCompletelyCoveredNodeStyle; + } else { + node_coverage_style = kPartiallyCoveredNodeStyle; + } + } else { + node_coverage_style = kCompletelyCoveredNodeStyle; + } + } + std::string escaped_expr_text = EscapeSpecialCharacters(expr_text); + dot_graph += absl::StrFormat( + "%d [shape=record, %s, label=\"{<1> exprID: %d | <2> %s} | <3> %s\"];\n", + node_id, node_coverage_style, node_id, KindToString(node), + escaped_expr_text); + bool node_covered = stats.covered; if (node_covered) { report.covered_nodes++; @@ -116,8 +163,10 @@ void TraverseAndCalculateCoverage( } for (const auto* child : node.children()) { + dot_graph += absl::StrFormat("%d -> %d;\n", node_id, child->expr()->id()); TraverseAndCalculateCoverage(checked_expr, *child, stats_map, - log_unencountered, preceeding_tabs, report); + log_unencountered, preceeding_tabs, report, + dot_graph); } } @@ -150,8 +199,13 @@ CoverageIndex::CoverageReport CoverageIndex::GetCoverageReport() const { if (node_coverage_stats_.empty()) { return report; } + + std::string dot_graph = std::string(kDigraphHeader); TraverseAndCalculateCoverage(checked_expr_, navigable_ast_.Root(), - node_coverage_stats_, true, "", report); + node_coverage_stats_, true, "", report, + dot_graph); + dot_graph += "}\n"; + report.dot_graph = dot_graph; report.cel_expression = google::api::expr::Unparse(checked_expr_).value_or(""); return report; diff --git a/testing/testrunner/coverage_index.h b/testing/testrunner/coverage_index.h index 53121d0f5..ae9cf902f 100644 --- a/testing/testrunner/coverage_index.h +++ b/testing/testrunner/coverage_index.h @@ -21,12 +21,20 @@ #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "common/value.h" #include "eval/public/cel_expression.h" #include "runtime/runtime.h" #include "tools/navigable_ast.h" namespace cel::test { +inline constexpr absl::string_view kDigraphHeader = "digraph {\n"; +inline constexpr absl::string_view kUncoveredNodeStyle = + R"(color="indianred2", style=filled)"; +inline constexpr absl::string_view kPartiallyCoveredNodeStyle = + R"(color="lightyellow", style=filled)"; +inline constexpr absl::string_view kCompletelyCoveredNodeStyle = + R"(color="lightgreen", style=filled)"; // `CoverageIndex` is a utility for tracking expression coverage based on the // Abstract Syntax Tree (AST) of a `cel::expr::CheckedExpr`. @@ -65,6 +73,7 @@ class CoverageIndex { int64_t covered_boolean_outcomes = 0; std::vector unencountered_nodes; std::vector unencountered_branches; + std::string dot_graph; }; // Initializes the coverage index with the given checked expression. diff --git a/testing/testrunner/runner_lib_test.cc b/testing/testrunner/runner_lib_test.cc index c4b7afefa..35f14407d 100644 --- a/testing/testrunner/runner_lib_test.cc +++ b/testing/testrunner/runner_lib_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "testing/testrunner/runner_lib.h" +#include #include #include #include @@ -45,6 +46,7 @@ #include "testing/testrunner/cel_test_context.h" #include "testing/testrunner/coverage_index.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "cel/expr/conformance/test/suite.pb.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" @@ -60,6 +62,10 @@ using ::cel::expr::conformance::test::TestCase; using ::cel::expr::CheckedExpr; using ::google::api::expr::runtime::CelExpressionBuilder; using ValueProto = ::cel::expr::Value; +using ::testing::EndsWith; +using ::testing::HasSubstr; +using ::testing::Not; +using ::testing::StartsWith; template T ParseTextProtoOrDie(absl::string_view text_proto) { @@ -68,6 +74,16 @@ T ParseTextProtoOrDie(absl::string_view text_proto) { return result; } +int CountSubstrings(absl::string_view text, absl::string_view substr) { + int count = 0; + size_t pos = 0; + while ((pos = text.find(substr, pos)) != absl::string_view::npos) { + ++count; + ++pos; + } + return count; +} + absl::StatusOr> CreateBasicCompiler() { CEL_ASSIGN_OR_RETURN( std::unique_ptr builder, @@ -660,11 +676,11 @@ TEST(CoverageTest, RuntimeCoverage) { EXPECT_THAT( report.unencountered_branches, ::testing::ElementsAre( - ::testing::HasSubstr("\nExpression ID 7 ('x > 1 && y > 1'): Never " - "evaluated to 'true'"), - ::testing::HasSubstr( + HasSubstr("\nExpression ID 7 ('x > 1 && y > 1'): Never " + "evaluated to 'true'"), + HasSubstr( "\n\t\tExpression ID 2 ('x > 1'): Never evaluated to 'false'"), - ::testing::HasSubstr( + HasSubstr( "\n\t\tExpression ID 5 ('y > 1'): Never evaluated to 'true'"))); } @@ -719,11 +735,196 @@ TEST(CoverageTest, BuilderCoverage) { EXPECT_EQ(report.branches, 6); EXPECT_EQ(report.covered_boolean_outcomes, 2); EXPECT_THAT(report.unencountered_nodes, - ::testing::UnorderedElementsAre(::testing::HasSubstr("y > 1"))); - EXPECT_THAT(report.unencountered_branches, - ::testing::UnorderedElementsAre( - ::testing::HasSubstr("Never evaluated to 'true'"), - ::testing::HasSubstr("Never evaluated to 'true'"))); + ::testing::UnorderedElementsAre(HasSubstr("y > 1"))); + EXPECT_THAT( + report.unencountered_branches, + ::testing::UnorderedElementsAre(HasSubstr("Never evaluated to 'true'"), + HasSubstr("Never evaluated to 'true'"))); +} + +TEST(CoverageTest, DotGraphIsGeneratedForRuntime) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("x", cel::IntType())), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("y", cel::IntType())), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("x > 1 && y > 1")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 2 } } + } + input { + key: "y" + value { value { int64_value: 0 } } + } + output { result_value { bool_value: false } } + )pb"); + + CoverageIndex coverage_index; + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + ASSERT_THAT(EnableCoverageInRuntime(*const_cast(runtime.get()), + coverage_index), + absl_testing::IsOk()); + + std::unique_ptr context = CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr(checked_expr)}); + TestRunner test_runner(std::move(context)); + coverage_index.Init(checked_expr); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); + + CoverageIndex::CoverageReport report = coverage_index.GetCoverageReport(); + + absl::string_view dot_graph = report.dot_graph; + + // Check for graph structure + EXPECT_THAT(dot_graph, StartsWith(kDigraphHeader)); + EXPECT_THAT(dot_graph, EndsWith("}\n")); + EXPECT_THAT(dot_graph, HasSubstr("->")); + EXPECT_THAT(dot_graph, HasSubstr("shape=record")); + + // Check for the existence of complete labels for key nodes, using the actual + // expression IDs from the build log. + EXPECT_THAT(dot_graph, HasSubstr("label=\"{<1> exprID: 7 | <2> Call Node} | " + "<3> x \\> 1 && y \\> 1\"")); + EXPECT_THAT( + dot_graph, + HasSubstr("label=\"{<1> exprID: 2 | <2> Call Node} | <3> x \\> 1\"")); + EXPECT_THAT( + dot_graph, + HasSubstr("label=\"{<1> exprID: 5 | <2> Call Node} | <3> y \\> 1\"")); + + // Check for coverage styles + EXPECT_THAT(dot_graph, HasSubstr(kCompletelyCoveredNodeStyle)); + EXPECT_THAT(dot_graph, HasSubstr(kPartiallyCoveredNodeStyle)); + EXPECT_THAT(dot_graph, Not(HasSubstr(kUncoveredNodeStyle))); +} + +TEST(CoverageTest, DotGraphIsGeneratedForComprehension) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("[1, 2, 3].all(i, i > 0)")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Test case expects 'true' since all elements are > 0. + TestCase test_case = ParseTextProtoOrDie(R"pb( + output { result_value { bool_value: true } } + )pb"); + + CoverageIndex coverage_index; + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + ASSERT_THAT(EnableCoverageInRuntime(*const_cast(runtime.get()), + coverage_index), + absl_testing::IsOk()); + + std::unique_ptr context = CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr(checked_expr)}); + TestRunner test_runner(std::move(context)); + coverage_index.Init(checked_expr); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); + + CoverageIndex::CoverageReport report = coverage_index.GetCoverageReport(); + absl::string_view dot_graph = report.dot_graph; + + // Assert that the specific kinds for comprehension nodes are present in the + // generated graph. + EXPECT_THAT(dot_graph, HasSubstr("IterRange")); + EXPECT_THAT(dot_graph, HasSubstr("AccuInit")); + EXPECT_THAT(dot_graph, HasSubstr("LoopCondition")); + EXPECT_THAT(dot_graph, HasSubstr("LoopStep")); + EXPECT_THAT(dot_graph, HasSubstr("Result")); + + // The expression is fully evaluated, so no nodes should be uncovered. + EXPECT_THAT(dot_graph, Not(HasSubstr(kUncoveredNodeStyle))); +} + +TEST(CoverageTest, PartiallyCoveredBooleanNodeIsStyledCorrectly) { + // This test is designed to kill a mutant that incorrectly styles partially + // covered boolean nodes as completely covered. It uses a short-circuiting + // expression to ensure that some boolean nodes are only evaluated one way + // (e.g., only to 'true'), making them partially covered. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("x", cel::IntType())), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("y", cel::IntType())), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + ASSERT_OK_AND_ASSIGN( + cel::ValidationResult validation_result, + compiler->Compile("{'sum': x + y, 'literal': 3}.sum == 3 || x == y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 1 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { result_value { bool_value: true } } + )pb"); + + CoverageIndex coverage_index; + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + ASSERT_THAT(EnableCoverageInRuntime(*const_cast(runtime.get()), + coverage_index), + absl_testing::IsOk()); + std::unique_ptr context = CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr(checked_expr)}); + TestRunner test_runner(std::move(context)); + coverage_index.Init(checked_expr); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); + + CoverageIndex::CoverageReport report = coverage_index.GetCoverageReport(); + + // With x=1, y=2, the left side of '||' is true, so the right side ('x == y') + // is short-circuited and never evaluated. + // - The '||' node and the '==' node are partially covered (only 'true'). + // - The 'x == y' branch (and its children) are uncovered. + // - All other evaluated nodes are fully covered. + EXPECT_EQ(CountSubstrings(report.dot_graph, kPartiallyCoveredNodeStyle), 2); + EXPECT_EQ(CountSubstrings(report.dot_graph, kUncoveredNodeStyle), 3); + EXPECT_EQ(CountSubstrings(report.dot_graph, kCompletelyCoveredNodeStyle), 9); } } // namespace diff --git a/testing/testrunner/user_tests/BUILD b/testing/testrunner/user_tests/BUILD index f2bf80628..2e3e28a83 100644 --- a/testing/testrunner/user_tests/BUILD +++ b/testing/testrunner/user_tests/BUILD @@ -85,6 +85,7 @@ cel_cc_test( cel_cc_test( name = "raw_expression_test_with_custom_test_suite", + enable_coverage = True, deps = [ ":raw_expression_user_test", ],