diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 33541f822..c5a97ae57 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -5,6 +5,7 @@ DEFAULT_VISIBILITY = [ "//eval:__subpackages__", "//runtime:__subpackages__", "//extensions:__subpackages__", + "//testing:__subpackages__", ] # This package contains code diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD index 975b5884d..0e8be6ca9 100644 --- a/testing/testrunner/BUILD +++ b/testing/testrunner/BUILD @@ -44,6 +44,8 @@ cc_library( "//internal:testing_no_main", "//runtime", "//runtime:activation", + "//tools:cel_unparser", + "//tools:navigable_ast", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -80,6 +82,7 @@ cc_test( deps = [ ":cel_expression_source", ":cel_test_context", + ":coverage_index", ":runner_lib", "//checker:type_checker_builder", "//checker:validation_result", @@ -115,14 +118,18 @@ cc_library( deps = [ ":cel_test_context", ":cel_test_factories", + ":coverage_index", ":runner_lib", + "//eval/public:cel_expression", "//internal:testing_no_main", + "//runtime", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//src/google/protobuf/io", @@ -135,3 +142,27 @@ cc_library( hdrs = ["cel_expression_source.h"], deps = ["@com_google_cel_spec//proto/cel/expr:checked_cc_proto"], ) + +cc_library( + name = "coverage_index", + srcs = ["coverage_index.cc"], + hdrs = ["coverage_index.h"], + deps = [ + "//common:ast", + "//common:value", + "//eval/compiler:cel_expression_builder_flat_impl", + "//eval/compiler:instrumentation", + "//eval/public:cel_expression", + "//internal:casts", + "//runtime", + "//runtime/internal:runtime_impl", + "//tools:cel_unparser", + "//tools:navigable_ast", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + ], +) diff --git a/testing/testrunner/cel_cc_test.bzl b/testing/testrunner/cel_cc_test.bzl index 07581ce3c..bbbdd51a1 100644 --- a/testing/testrunner/cel_cc_test.bzl +++ b/testing/testrunner/cel_cc_test.bzl @@ -21,6 +21,7 @@ def cel_cc_test( test_suite = "", filegroup = "", deps = [], + enable_coverage = False, test_data_path = "", data = []): """trigger the cc impl of the CEL test runner. @@ -36,6 +37,7 @@ def cel_cc_test( expression. deps: list of dependencies for the cc_test rule. data: list of data dependencies for the cc_test rule. + enable_coverage: bool whether to enable coverage collection. test_data_path: absolute path of the directory containing the test files. This is needed only if the test files are not located in the same directory as the BUILD file. """ @@ -48,6 +50,8 @@ def cel_cc_test( test_suite = test_data_path + "/" + test_suite args.append("--test_suite_path=" + test_suite) + args.append("--collect_coverage=" + str(enable_coverage)) + cc_test( name = name, data = data, diff --git a/testing/testrunner/coverage_index.cc b/testing/testrunner/coverage_index.cc new file mode 100644 index 000000000..bdc3febc4 --- /dev/null +++ b/testing/testrunner/coverage_index.cc @@ -0,0 +1,192 @@ +// Copyright 2025 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testing/testrunner/coverage_index.h" + +#include +#include +#include + +#include "cel/expr/syntax.pb.h" +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "common/ast.h" +#include "common/value.h" +#include "eval/compiler/cel_expression_builder_flat_impl.h" +#include "eval/compiler/instrumentation.h" +#include "eval/public/cel_expression.h" +#include "internal/casts.h" +#include "runtime/internal/runtime_impl.h" +#include "runtime/runtime.h" +#include "tools/cel_unparser.h" +#include "tools/navigable_ast.h" + +namespace cel::test { +namespace { + +using ::cel::expr::CheckedExpr; +using ::cel::expr::Type; +using ::google::api::expr::runtime::CelExpressionBuilder; +using ::google::api::expr::runtime::Instrumentation; +using ::google::api::expr::runtime::InstrumentationFactory; + +const Type* absl_nullable FindCheckerType(const CheckedExpr& expr, + int64_t expr_id) { + if (auto it = expr.type_map().find(expr_id); it != expr.type_map().end()) { + return &it->second; + } + return nullptr; +} + +bool InferredBooleanNode(const CheckedExpr& checked_expr, + const NavigableProtoAstNode& node) { + int64_t node_id = node.expr()->id(); + const auto* checker_type = FindCheckerType(checked_expr, node_id); + if (checker_type != nullptr) { + return checker_type->has_primitive() && + checker_type->primitive() == Type::BOOL; + } + + return false; +} + +void TraverseAndCalculateCoverage( + const CheckedExpr& checked_expr, const NavigableProtoAstNode& node, + const absl::flat_hash_map& + stats_map, + bool log_unencountered, std::string preceeding_tabs, + CoverageIndex::CoverageReport& report) { + int64_t node_id = node.expr()->id(); + + const CoverageIndex::NodeCoverageStats& stats = stats_map.at(node_id); + report.nodes++; + + absl::StatusOr unparsed = + google::api::expr::Unparse(*node.expr()); + std::string expr_text = unparsed.ok() ? *unparsed : "unparse_failed"; + + bool is_interesting_bool_node = + stats.is_boolean_node && !node.expr()->has_const_expr() && + (!node.expr()->has_call_expr() || + node.expr()->call_expr().function() != "cel.@block"); + + bool node_covered = stats.covered; + if (node_covered) { + report.covered_nodes++; + } else if (log_unencountered) { + if (is_interesting_bool_node) { + report.unencountered_nodes.push_back( + absl::StrCat("Expression ID ", node_id, " ('", expr_text, "')")); + } + log_unencountered = false; + } + + if (is_interesting_bool_node) { + report.branches += 2; + if (stats.has_true_branch) { + report.covered_boolean_outcomes++; + } else if (log_unencountered) { + report.unencountered_branches.push_back( + absl::StrCat("\n", preceeding_tabs, "Expression ID ", node_id, " ('", + expr_text, "'): Never evaluated to 'true'")); + preceeding_tabs += "\t\t"; + } + if (stats.has_false_branch) { + report.covered_boolean_outcomes++; + } else if (log_unencountered) { + report.unencountered_branches.push_back( + absl::StrCat("\n", preceeding_tabs, "Expression ID ", node_id, " ('", + expr_text, "'): Never evaluated to 'false'")); + preceeding_tabs += "\t\t"; + } + } + + for (const auto* child : node.children()) { + TraverseAndCalculateCoverage(checked_expr, *child, stats_map, + log_unencountered, preceeding_tabs, report); + } +} + +} // namespace + +void CoverageIndex::RecordCoverage(int64_t node_id, const cel::Value& value) { + NodeCoverageStats& stats = node_coverage_stats_[node_id]; + stats.covered = true; + if (node_coverage_stats_[node_id].is_boolean_node) { + if (value.AsBool()->NativeValue()) { + stats.has_true_branch = true; + } else { + stats.has_false_branch = true; + } + } +} + +void CoverageIndex::Init(const cel::expr::CheckedExpr& checked_expr) { + checked_expr_ = checked_expr; + navigable_ast_ = NavigableProtoAst::Build(checked_expr_.expr()); + for (const auto& node : navigable_ast_.Root().DescendantsPreorder()) { + NodeCoverageStats stats; + stats.is_boolean_node = InferredBooleanNode(checked_expr_, node); + node_coverage_stats_[node.expr()->id()] = stats; + } +} + +CoverageIndex::CoverageReport CoverageIndex::GetCoverageReport() const { + CoverageReport report; + if (node_coverage_stats_.empty()) { + return report; + } + TraverseAndCalculateCoverage(checked_expr_, navigable_ast_.Root(), + node_coverage_stats_, true, "", report); + report.cel_expression = + google::api::expr::Unparse(checked_expr_).value_or(""); + return report; +} + +InstrumentationFactory InstrumentationFactoryForCoverage( + CoverageIndex& coverage_index) { + return [&](const cel::Ast& ast) -> Instrumentation { + return [&](int64_t node_id, const cel::Value& value) -> absl::Status { + coverage_index.RecordCoverage(node_id, value); + return absl::OkStatus(); + }; + }; +} + +absl::Status EnableCoverageInRuntime(cel::Runtime& runtime, + CoverageIndex& coverage_index) { + auto& runtime_impl = + cel::internal::down_cast(runtime); + runtime_impl.expr_builder().AddProgramOptimizer( + google::api::expr::runtime::CreateInstrumentationExtension( + InstrumentationFactoryForCoverage(coverage_index))); + return absl::OkStatus(); +} + +absl::Status EnableCoverageInCelExpressionBuilder( + CelExpressionBuilder& cel_expression_builder, + CoverageIndex& coverage_index) { + auto& cel_expression_builder_impl = cel::internal::down_cast< + google::api::expr::runtime::CelExpressionBuilderFlatImpl&>( + cel_expression_builder); + cel_expression_builder_impl.flat_expr_builder().AddProgramOptimizer( + google::api::expr::runtime::CreateInstrumentationExtension( + InstrumentationFactoryForCoverage(coverage_index))); + return absl::OkStatus(); +} + +} // namespace cel::test diff --git a/testing/testrunner/coverage_index.h b/testing/testrunner/coverage_index.h new file mode 100644 index 000000000..53121d0f5 --- /dev/null +++ b/testing/testrunner/coverage_index.h @@ -0,0 +1,111 @@ +// Copyright 2025 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_COVERAGE_INDEX_H_ +#define THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_COVERAGE_INDEX_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "common/value.h" +#include "eval/public/cel_expression.h" +#include "runtime/runtime.h" +#include "tools/navigable_ast.h" + +namespace cel::test { + +// `CoverageIndex` is a utility for tracking expression coverage based on the +// Abstract Syntax Tree (AST) of a `cel::expr::CheckedExpr`. +// +// To use `CoverageIndex`, it must first be initialized with a +// `cel::expr::CheckedExpr` using the `Init` method. This allows the +// index to build up a representation of all the nodes and potential boolean +// branches within the expression. +// +// The `CoverageIndex` is then integrated with the CEL evaluation process. +// This is done by enabling coverage either in a `cel::Runtime` or a +// `google::api::expr::runtime::CelExpressionBuilder` using the provided helper +// functions (`EnableCoverageInRuntime` or +// `EnableCoverageInCelExpressionBuilder`). When integrated, the CEL evaluation +// engine will call `RecordCoverage` for each visited expression node, allowing +// `CoverageIndex` to track which parts of the expression were executed and, +// for boolean-producing nodes, which branches were taken (true/false). +// +// After evaluation, a `CoverageReport` can be generated, summarizing the +// executed nodes and branches, and highlighting any unencountered parts of +// the expression. +class CoverageIndex { + public: + struct NodeCoverageStats { + bool is_boolean_node = false; + bool covered = false; + bool has_true_branch = false; + bool has_false_branch = false; + }; + + struct CoverageReport { + std::string cel_expression; + int64_t nodes = 0; + int64_t covered_nodes = 0; + int64_t branches = 0; + int64_t covered_boolean_outcomes = 0; + std::vector unencountered_nodes; + std::vector unencountered_branches; + }; + + // Initializes the coverage index with the given checked expression. + // + // The coverage index will be initialized with an entry for each node in the + // AST. + void Init(const cel::expr::CheckedExpr& checked_expr); + + // Records coverage for the given node. + // + // The coverage index will be updated with the coverage information for the + // given node. + void RecordCoverage(int64_t node_id, const cel::Value& value); + + // Returns a coverage report for the given checked expression. + CoverageReport GetCoverageReport() const; + + private: + absl::flat_hash_map node_coverage_stats_; + NavigableProtoAst navigable_ast_; + cel::expr::CheckedExpr checked_expr_; +}; + +// Enables coverage tracking within the provided `cel::Runtime`. +// Note: This function ties the `runtime` instance to a single expression. +// Do not reuse this `runtime` instance with multiple expressions when coverage +// is enabled, as the `coverage_index` will accumulate results across different +// expressions, leading to incorrect coverage reports. +absl::Status EnableCoverageInCelExpressionBuilder( + google::api::expr::runtime::CelExpressionBuilder& cel_expression_builder, + CoverageIndex& coverage_index); + +// Enables coverage tracking within the provided `CelExpressionBuilder`. +// Note: This function ties the `cel_expression_builder` instance to a single +// expression. Do not reuse this `cel_expression_builder` instance with +// multiple expressions when coverage is enabled, as the `coverage_index` will +// accumulate results across different expressions, leading to incorrect +// coverage reports. +absl::Status EnableCoverageInRuntime(cel::Runtime& runtime, + CoverageIndex& coverage_index); + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_COVERAGE_INDEX_H_ diff --git a/testing/testrunner/resources/simple_tests.textproto b/testing/testrunner/resources/simple_tests.textproto index 46275312f..7add08851 100644 --- a/testing/testrunner/resources/simple_tests.textproto +++ b/testing/testrunner/resources/simple_tests.textproto @@ -19,16 +19,7 @@ sections: { } output { result_value { - map_value { - entries { - key { string_value: "literal" } - value { int64_value: 3 } - } - entries { - key { string_value: "sum" } - value { int64_value: 3 } - } - } + bool_value: true } } } @@ -45,16 +36,7 @@ sections: { } output { result_value { - map_value { - entries { - key { string_value: "literal" } - value { int64_value: 3 } - } - entries { - key { string_value: "sum" } - value { int64_value: 7 } - } - } + bool_value: false } } } diff --git a/testing/testrunner/runner_bin.cc b/testing/testrunner/runner_bin.cc index a307d45c0..88a46bba5 100644 --- a/testing/testrunner/runner_bin.cc +++ b/testing/testrunner/runner_bin.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,14 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "eval/public/cel_expression.h" #include "internal/testing.h" +#include "runtime/runtime.h" #include "testing/testrunner/cel_test_context.h" #include "testing/testrunner/cel_test_factories.h" +#include "testing/testrunner/coverage_index.h" #include "testing/testrunner/runner_lib.h" #include "cel/expr/conformance/test/suite.pb.h" #include "google/protobuf/io/zero_copy_stream_impl.h" @@ -39,11 +45,76 @@ ABSL_FLAG(std::string, test_suite_path, "", "The path to the file containing the test suite to run."); +ABSL_FLAG(bool, collect_coverage, false, "Whether to collect code coverage."); + namespace { using ::cel::expr::conformance::test::TestCase; using ::cel::expr::conformance::test::TestSuite; +using ::cel::test::CelTestContext; +using ::cel::test::CoverageIndex; using ::cel::test::TestRunner; +using ::google::api::expr::runtime::CelExpressionBuilder; + +class CoverageReportingEnvironment : public testing::Environment { + public: + explicit CoverageReportingEnvironment(CoverageIndex& coverage_index) + : coverage_index_(coverage_index) {} + + void TearDown() override { + CoverageIndex::CoverageReport coverage_report = + coverage_index_.GetCoverageReport(); + testing::Test::RecordProperty("CEL Expression", + coverage_report.cel_expression); + std::cout << "CEL Expression: " << coverage_report.cel_expression; + + if (coverage_report.nodes == 0) { + testing::Test::RecordProperty("CEL Coverage", "No coverage stats found"); + std::cout << "CEL Coverage: " << "No coverage stats found"; + return; + } + + // Log Node Coverage results + double node_coverage = static_cast(coverage_report.covered_nodes) / + static_cast(coverage_report.nodes) * 100.0; + std::string node_coverage_string = + absl::StrFormat("%.2f%% (%d out of %d nodes covered)", node_coverage, + coverage_report.covered_nodes, coverage_report.nodes); + testing::Test::RecordProperty("AST Node Coverage", node_coverage_string); + std::cout << "AST Node Coverage: " << node_coverage_string; + if (!coverage_report.unencountered_nodes.empty()) { + testing::Test::RecordProperty( + "Interesting Unencountered Nodes", + absl::StrJoin(coverage_report.unencountered_nodes, "\n")); + std::cout << "Interesting Unencountered Nodes: " + << absl::StrJoin(coverage_report.unencountered_nodes, "\n"); + } + // Log Branch Coverage results + double branch_coverage = 0.0; + if (coverage_report.branches > 0) { + branch_coverage = + static_cast(coverage_report.covered_boolean_outcomes) / + static_cast(coverage_report.branches) * 100.0; + } + std::string branch_coverage_string = absl::StrFormat( + "%.2f%% (%d out of %d branch outcomes covered)", branch_coverage, + coverage_report.covered_boolean_outcomes, coverage_report.branches); + testing::Test::RecordProperty("AST Branch Coverage", + branch_coverage_string); + std::cout << "AST Branch Coverage: " << branch_coverage_string; + if (!coverage_report.unencountered_branches.empty()) { + testing::Test::RecordProperty( + "Interesting Unencountered Branch Paths", + absl::StrJoin(coverage_report.unencountered_branches, "\n")); + std::cout << "Interesting Unencountered Branch Paths: " + << absl::StrJoin(coverage_report.unencountered_branches, + "\n"); + } + } + + private: + cel::test::CoverageIndex& coverage_index_; +}; class CelTest : public testing::Test { public: @@ -115,15 +186,45 @@ int main(int argc, char** argv) { // Create a test context using the factory function returned by the global // factory function provider which was initialized by the user. - absl::StatusOr> cel_test_context = + absl::StatusOr> cel_test_context = cel::test::internal::GetCelTestContextFactory()(); if (!cel_test_context.ok()) { ABSL_LOG(FATAL) << "Failed to create CEL test context: " << cel_test_context.status(); } + + cel::test::CoverageIndex coverage_index; + if (absl::GetFlag(FLAGS_collect_coverage)) { + if (cel_test_context.value()->runtime() != nullptr) { + ABSL_CHECK_OK(cel::test::EnableCoverageInRuntime( + const_cast(*cel_test_context.value()->runtime()), + coverage_index)); + } else if (cel_test_context.value()->cel_expression_builder() != nullptr) { + ABSL_CHECK_OK(cel::test::EnableCoverageInCelExpressionBuilder( + const_cast( + *cel_test_context.value()->cel_expression_builder()), + coverage_index)); + } + } + auto test_runner = std::make_shared(std::move(cel_test_context.value())); ABSL_CHECK_OK(RegisterTests(GetTestSuite(), test_runner)); + // Make sure the checked expression exists during the entire test run since + // the ast references it during coverage collection at teardown. + absl::StatusOr checked_expr = + test_runner->GetCheckedExpr(); + if (!checked_expr.ok()) { + ABSL_LOG(FATAL) << "Failed to get checked expression: " + << checked_expr.status(); + } + + if (absl::GetFlag(FLAGS_collect_coverage)) { + coverage_index.Init(*checked_expr); + testing::AddGlobalTestEnvironment( + new CoverageReportingEnvironment(coverage_index)); + } + return RUN_ALL_TESTS(); } diff --git a/testing/testrunner/runner_lib.h b/testing/testrunner/runner_lib.h index 40a5c50f4..a04602d49 100644 --- a/testing/testrunner/runner_lib.h +++ b/testing/testrunner/runner_lib.h @@ -36,6 +36,9 @@ class TestRunner { // assertions against the expected result. void RunTest(const cel::expr::conformance::test::TestCase& test_case); + // Returns the checked expression for the test case. + absl::StatusOr GetCheckedExpr() const; + private: absl::StatusOr EvalWithRuntime( const cel::expr::CheckedExpr& checked_expr, @@ -55,8 +58,6 @@ class TestRunner { const cel::expr::conformance::test::TestOutput& output, google::protobuf::Arena* arena); - absl::StatusOr GetCheckedExpr() const; - void AssertError(const cel::Value& computed, const cel::expr::conformance::test::TestOutput& output); diff --git a/testing/testrunner/runner_lib_test.cc b/testing/testrunner/runner_lib_test.cc index f63952b2c..c4b7afefa 100644 --- a/testing/testrunner/runner_lib_test.cc +++ b/testing/testrunner/runner_lib_test.cc @@ -43,6 +43,7 @@ #include "runtime/standard_runtime_builder_factory.h" #include "testing/testrunner/cel_expression_source.h" #include "testing/testrunner/cel_test_context.h" +#include "testing/testrunner/coverage_index.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -604,5 +605,126 @@ TEST(TestRunnerStandaloneTest, BasicTestFailsWhenExpectingErrorButGotValue) { EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), "Expected error but got value"); } + +TEST(CoverageTest, RuntimeCoverage) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("x", cel::IntType())), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("y", cel::IntType())), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("x > 1 && y > 1")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 2 } } + } + input { + key: "y" + value { value { int64_value: 0 } } + } + output { result_value { bool_value: false } } + )pb"); + + CoverageIndex coverage_index; + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + ASSERT_THAT(EnableCoverageInRuntime(*const_cast(runtime.get()), + coverage_index), + absl_testing::IsOk()); + + auto context = CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr(checked_expr)}); + TestRunner test_runner(std::move(context)); + coverage_index.Init(checked_expr); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); + + CoverageIndex::CoverageReport report = coverage_index.GetCoverageReport(); + EXPECT_GT(report.nodes, 0); + EXPECT_GT(report.covered_nodes, 0); + EXPECT_EQ(report.branches, 6); + EXPECT_EQ(report.covered_boolean_outcomes, 3); + EXPECT_THAT( + report.unencountered_branches, + ::testing::ElementsAre( + ::testing::HasSubstr("\nExpression ID 7 ('x > 1 && y > 1'): Never " + "evaluated to 'true'"), + ::testing::HasSubstr( + "\n\t\tExpression ID 2 ('x > 1'): Never evaluated to 'false'"), + ::testing::HasSubstr( + "\n\t\tExpression ID 5 ('y > 1'): Never evaluated to 'true'"))); +} + +TEST(CoverageTest, BuilderCoverage) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("x", cel::IntType())), + absl_testing::IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("y", cel::IntType())), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("x > 1 && y > 1")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 0 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { result_value { bool_value: false } } + )pb"); + + CoverageIndex coverage_index; + ASSERT_OK_AND_ASSIGN(std::unique_ptr builder, + CreateTestCelExpressionBuilder()); + ASSERT_THAT(EnableCoverageInCelExpressionBuilder(*builder, coverage_index), + absl_testing::IsOk()); + + auto context = CelTestContext::CreateFromCelExpressionBuilder( + std::move(builder), + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr(checked_expr)}); + TestRunner test_runner(std::move(context)); + coverage_index.Init(checked_expr); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); + + CoverageIndex::CoverageReport report = coverage_index.GetCoverageReport(); + EXPECT_GT(report.nodes, 0); + EXPECT_GT(report.covered_nodes, 0); + EXPECT_EQ(report.branches, 6); + EXPECT_EQ(report.covered_boolean_outcomes, 2); + EXPECT_THAT(report.unencountered_nodes, + ::testing::UnorderedElementsAre(::testing::HasSubstr("y > 1"))); + EXPECT_THAT(report.unencountered_branches, + ::testing::UnorderedElementsAre( + ::testing::HasSubstr("Never evaluated to 'true'"), + ::testing::HasSubstr("Never evaluated to 'true'"))); +} + } // namespace } // namespace cel::test diff --git a/testing/testrunner/user_tests/BUILD b/testing/testrunner/user_tests/BUILD index 436176f1c..f2bf80628 100644 --- a/testing/testrunner/user_tests/BUILD +++ b/testing/testrunner/user_tests/BUILD @@ -20,6 +20,7 @@ cc_library( "//internal:testing_descriptor_pool", "//runtime", "//runtime:runtime_builder", + "//runtime:runtime_options", "//runtime:standard_runtime_builder_factory", "//testing/testrunner:cel_expression_source", "//testing/testrunner:cel_test_context", @@ -63,6 +64,7 @@ cc_library( cel_cc_test( name = "simple_test", + enable_coverage = True, filegroup = "//testing/testrunner/resources", test_data_path = "//testing/testrunner/resources", test_suite = "simple_tests.textproto", @@ -73,6 +75,7 @@ cel_cc_test( cel_cc_test( name = "simple_test_with_custom_test_suite", + enable_coverage = True, filegroup = "//testing/testrunner/resources", test_data_path = "//testing/testrunner/resources", deps = [ diff --git a/testing/testrunner/user_tests/simple.cc b/testing/testrunner/user_tests/simple.cc index e199f6d17..7e3cafba6 100644 --- a/testing/testrunner/user_tests/simple.cc +++ b/testing/testrunner/user_tests/simple.cc @@ -68,20 +68,7 @@ CEL_REGISTER_TEST_SUITE_FACTORY([]() { key: "y" value { value { int64_value: 2 } } } - output { - result_value { - map_value { - entries { - key { string_value: "literal" } - value { int64_value: 3 } - } - entries { - key { string_value: "sum" } - value { int64_value: 3 } - } - } - } - } + output { result_value { bool_value: true } } } } )pb"); @@ -105,8 +92,9 @@ CEL_REGISTER_TEST_CONTEXT_FACTORY( builder->Build()); // Compile the expression. - CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, - compiler->Compile("{'sum': x + y, 'literal': 3}")); + CEL_ASSIGN_OR_RETURN( + cel::ValidationResult validation_result, + compiler->Compile("{'sum': x + y, 'literal': 3}.sum == 3 || x == y")); CheckedExpr checked_expr; CEL_RETURN_IF_ERROR( cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr));