From f204194bb5d14876b13c71201657c9f7c4c1087c Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 26 Oct 2025 21:52:35 -0700 Subject: [PATCH] Enable coverage collection via the runner library for a single test. PiperOrigin-RevId: 824348140 --- testing/testrunner/BUILD | 20 +++- testing/testrunner/cel_test_context.h | 8 ++ testing/testrunner/coverage_reporting.cc | 124 +++++++++++++++++++++ testing/testrunner/coverage_reporting.h | 43 +++++++ testing/testrunner/runner_bin.cc | 136 ++++------------------- testing/testrunner/runner_lib.cc | 26 +++++ testing/testrunner/runner_lib.h | 19 ++++ 7 files changed, 260 insertions(+), 116 deletions(-) create mode 100644 testing/testrunner/coverage_reporting.cc create mode 100644 testing/testrunner/coverage_reporting.h diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD index 949dc56e7..e1b3139cc 100644 --- a/testing/testrunner/BUILD +++ b/testing/testrunner/BUILD @@ -36,6 +36,8 @@ cc_library( deps = [ ":cel_expression_source", ":cel_test_context", + ":coverage_index", + ":coverage_reporting", "//checker:validation_result", "//common:ast", "//common:ast_proto", @@ -49,10 +51,9 @@ cc_library( "//internal:testing_no_main", "//runtime", "//runtime:activation", - "//tools:cel_unparser", - "//tools:navigable_ast", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", @@ -120,6 +121,19 @@ cc_test( ], ) +cc_library( + name = "coverage_reporting", + srcs = ["coverage_reporting.cc"], + hdrs = ["coverage_reporting.h"], + deps = [ + ":coverage_index", + "//internal:testing_no_main", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + ], +) + cc_library( name = "runner", srcs = ["runner_bin.cc"], @@ -128,6 +142,7 @@ cc_library( ":cel_test_context", ":cel_test_factories", ":coverage_index", + ":coverage_reporting", ":runner_lib", "//eval/public:cel_expression", "//internal:status_macros", @@ -139,7 +154,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/testing/testrunner/cel_test_context.h b/testing/testrunner/cel_test_context.h index aa6aab3ac..0e0f21e28 100644 --- a/testing/testrunner/cel_test_context.h +++ b/testing/testrunner/cel_test_context.h @@ -102,6 +102,8 @@ class CelTestContext { return custom_bindings_; } + bool enable_coverage() const { return enable_coverage_; } + // Allows the runner to inject the expression source // parsed from command-line flags. void SetExpressionSource(CelExpressionSource source) { @@ -128,6 +130,9 @@ class CelTestContext { activation_factory_ = std::move(activation_factory); } + // Allows the runner to enable coverage collection. + void SetEnableCoverage(bool enable) { enable_coverage_ = enable; } + const CelActivationFactoryFn& activation_factory() const { return activation_factory_; } @@ -185,6 +190,9 @@ class CelTestContext { CelActivationFactoryFn activation_factory_; AssertFn assert_fn_; + + // Whether to enable coverage collection. + bool enable_coverage_ = false; }; } // namespace cel::test diff --git a/testing/testrunner/coverage_reporting.cc b/testing/testrunner/coverage_reporting.cc new file mode 100644 index 000000000..d37386cc3 --- /dev/null +++ b/testing/testrunner/coverage_reporting.cc @@ -0,0 +1,124 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testing/testrunner/coverage_reporting.h" + +#include +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "internal/testing.h" +#include "testing/testrunner/coverage_index.h" + +namespace cel::test { +void CoverageReportingEnvironment::TearDown() { + CoverageIndex::CoverageReport coverage_report = + coverage_index_.GetCoverageReport(); + testing::Test::RecordProperty("CEL Expression", + coverage_report.cel_expression); + std::cout << "CEL Expression: " << coverage_report.cel_expression; + if (coverage_report.nodes == 0) { + testing::Test::RecordProperty("CEL Coverage", "No coverage stats found"); + std::cout << "CEL Coverage: " << "No coverage stats found"; + return; + } + + // Log Node Coverage results + double node_coverage = static_cast(coverage_report.covered_nodes) / + static_cast(coverage_report.nodes) * 100.0; + std::string node_coverage_string = + absl::StrFormat("%.2f%% (%d out of %d nodes covered)", node_coverage, + coverage_report.covered_nodes, coverage_report.nodes); + testing::Test::RecordProperty("AST Node Coverage", node_coverage_string); + std::cout << "AST Node Coverage: " << node_coverage_string; + if (!coverage_report.unencountered_nodes.empty()) { + testing::Test::RecordProperty( + "Interesting Unencountered Nodes", + absl::StrJoin(coverage_report.unencountered_nodes, "\n")); + std::cout << "Interesting Unencountered Nodes: " + << absl::StrJoin(coverage_report.unencountered_nodes, "\n"); + } + + // Log Branch Coverage results + double branch_coverage = 0.0; + if (coverage_report.branches > 0) { + branch_coverage = + static_cast(coverage_report.covered_boolean_outcomes) / + static_cast(coverage_report.branches) * 100.0; + } + std::string branch_coverage_string = absl::StrFormat( + "%.2f%% (%d out of %d branch outcomes covered)", branch_coverage, + coverage_report.covered_boolean_outcomes, coverage_report.branches); + testing::Test::RecordProperty("AST Branch Coverage", branch_coverage_string); + std::cout << "AST Branch Coverage: " << branch_coverage_string; + if (!coverage_report.unencountered_branches.empty()) { + testing::Test::RecordProperty( + "Interesting Unencountered Branch Paths", + absl::StrJoin(coverage_report.unencountered_branches, "\n")); + std::cout << "Interesting Unencountered Branch Paths: " + << absl::StrJoin(coverage_report.unencountered_branches, + "\n"); + } + if (!coverage_report.dot_graph.empty()) { + WriteDotGraphToArtifact(coverage_report.dot_graph); + } +} + +void CoverageReportingEnvironment::WriteDotGraphToArtifact( + absl::string_view dot_graph) { + // Save DOT graph to file in TEST_UNDECLARED_OUTPUTS_DIR or default dir + const char* outputs_dir_env = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR"); + // For non-Bazel/Blaze users, we write to a subdirectory under the current + // working directory. + // NOMUTANTS --cel_artifacts is for non-Bazel/Blaze users only so not + // needed to test in our case. + std::string outputs_dir = + (outputs_dir_env == nullptr) ? "cel_artifacts" : outputs_dir_env; + std::string coverage_dir = absl::StrCat(outputs_dir, "/cel_test_coverage"); + // Creates the directory to store CEL test coverage artifacts. + // The second argument, `0755`, sets the directory's permissions in octal + // format, which is a standard for file system operations. It grants: + // - Owner: read, write, and execute permissions (7 = 4+2+1). + // - Group: read and execute permissions (5 = 4+1). + // - Others: read and execute permissions (5 = 4+1). + // This gives the owner full control while allowing other users to access + // the generated artifacts. + int mkdir_result = mkdir(coverage_dir.c_str(), 0755); + // If mkdir fails, it sets the global 'errno' variable to an error code + // indicating the reason. We check this code to specifically ignore the + // EEXIST error, which just means the directory already exists (this is not + // a real failure we need to warn about). + if (mkdir_result == 0 || errno == EEXIST) { + std::string graph_path = absl::StrCat(coverage_dir, "/coverage_graph.txt"); + std::ofstream out(graph_path); + if (out.is_open()) { + out << dot_graph; + out.close(); + } else { + ABSL_LOG(WARNING) << "Failed to open file for writing: " << graph_path; + } + } else { + ABSL_LOG(WARNING) << "Failed to create directory: " << coverage_dir + << " (reason: " << strerror(errno) << ")"; + } +} +} // namespace cel::test diff --git a/testing/testrunner/coverage_reporting.h b/testing/testrunner/coverage_reporting.h new file mode 100644 index 000000000..2e1f4ad23 --- /dev/null +++ b/testing/testrunner/coverage_reporting.h @@ -0,0 +1,43 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_COVERAGE_REPORTING_H_ +#define THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_COVERAGE_REPORTING_H_ + +#include "absl/strings/string_view.h" +#include "internal/testing.h" +#include "testing/testrunner/coverage_index.h" + +namespace cel::test { +// A Google Test Environment that reports CEL coverage results in its TearDown +// phase. +// +// This class encapsulates the logic for calculating coverage statistics and +// logging them as test properties. +class CoverageReportingEnvironment : public testing::Environment { + public: + explicit CoverageReportingEnvironment(CoverageIndex& coverage_index) + : coverage_index_(coverage_index) {}; + + // Called by the Google Test framework after all tests have run. + void TearDown() override; + + private: + // Helper function to write the DOT graph to a test artifact file. + void WriteDotGraphToArtifact(absl::string_view dot_graph); + + CoverageIndex& coverage_index_; +}; +} // namespace cel::test +#endif // THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_COVERAGE_REPORTING_H_ diff --git a/testing/testrunner/runner_bin.cc b/testing/testrunner/runner_bin.cc index f28ad7037..c11908ca5 100644 --- a/testing/testrunner/runner_bin.cc +++ b/testing/testrunner/runner_bin.cc @@ -14,9 +14,6 @@ // This binary is a test runner for CEL tests. It is used to run CEL tests // written in the CEL test suite format. -#include -#include -#include #include #include #include @@ -35,8 +32,6 @@ #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "eval/public/cel_expression.h" #include "internal/status_macros.h" @@ -46,6 +41,7 @@ #include "testing/testrunner/cel_test_context.h" #include "testing/testrunner/cel_test_factories.h" #include "testing/testrunner/coverage_index.h" +#include "testing/testrunner/coverage_reporting.h" #include "testing/testrunner/runner_lib.h" #include "cel/expr/conformance/test/suite.pb.h" #include "google/protobuf/text_format.h" @@ -71,113 +67,6 @@ using ::cel::test::TestRunner; using ::cel::expr::CheckedExpr; using ::google::api::expr::runtime::CelExpressionBuilder; -class CoverageReportingEnvironment : public testing::Environment { - public: - explicit CoverageReportingEnvironment(CoverageIndex& coverage_index) - : coverage_index_(coverage_index) {} - - void TearDown() override { - CoverageIndex::CoverageReport coverage_report = - coverage_index_.GetCoverageReport(); - testing::Test::RecordProperty("CEL Expression", - coverage_report.cel_expression); - std::cout << "CEL Expression: " << coverage_report.cel_expression; - - if (coverage_report.nodes == 0) { - testing::Test::RecordProperty("CEL Coverage", "No coverage stats found"); - std::cout << "CEL Coverage: " << "No coverage stats found"; - return; - } - - // Log Node Coverage results - double node_coverage = static_cast(coverage_report.covered_nodes) / - static_cast(coverage_report.nodes) * 100.0; - std::string node_coverage_string = - absl::StrFormat("%.2f%% (%d out of %d nodes covered)", node_coverage, - coverage_report.covered_nodes, coverage_report.nodes); - testing::Test::RecordProperty("AST Node Coverage", node_coverage_string); - std::cout << "AST Node Coverage: " << node_coverage_string; - if (!coverage_report.unencountered_nodes.empty()) { - testing::Test::RecordProperty( - "Interesting Unencountered Nodes", - absl::StrJoin(coverage_report.unencountered_nodes, "\n")); - std::cout << "Interesting Unencountered Nodes: " - << absl::StrJoin(coverage_report.unencountered_nodes, "\n"); - } - // Log Branch Coverage results - double branch_coverage = 0.0; - if (coverage_report.branches > 0) { - branch_coverage = - static_cast(coverage_report.covered_boolean_outcomes) / - static_cast(coverage_report.branches) * 100.0; - } - std::string branch_coverage_string = absl::StrFormat( - "%.2f%% (%d out of %d branch outcomes covered)", branch_coverage, - coverage_report.covered_boolean_outcomes, coverage_report.branches); - testing::Test::RecordProperty("AST Branch Coverage", - branch_coverage_string); - std::cout << "AST Branch Coverage: " << branch_coverage_string; - if (!coverage_report.unencountered_branches.empty()) { - testing::Test::RecordProperty( - "Interesting Unencountered Branch Paths", - absl::StrJoin(coverage_report.unencountered_branches, "\n")); - std::cout << "Interesting Unencountered Branch Paths: " - << absl::StrJoin(coverage_report.unencountered_branches, - "\n"); - } - - if (!coverage_report.dot_graph.empty()) { - WriteDotGraphToArtifact(coverage_report.dot_graph); - } - } - - private: - void WriteDotGraphToArtifact(absl::string_view dot_graph) { - // Save DOT graph to file in TEST_UNDECLARED_OUTPUTS_DIR or default dir - const char* outputs_dir_env = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR"); - - // For non-Bazel/Blaze users, we write to a subdirectory under the current - // working directory. - // NOMUTANTS --cel_artifacts is for non-Bazel/Blaze users only so not - // needed to test in our case. - std::string outputs_dir = - (outputs_dir_env == nullptr) ? "cel_artifacts" : outputs_dir_env; - - std::string coverage_dir = absl::StrCat(outputs_dir, "/cel_test_coverage"); - - // Creates the directory to store CEL test coverage artifacts. - // The second argument, `0755`, sets the directory's permissions in octal - // format, which is a standard for file system operations. It grants: - // - Owner: read, write, and execute permissions (7 = 4+2+1). - // - Group: read and execute permissions (5 = 4+1). - // - Others: read and execute permissions (5 = 4+1). - // This gives the owner full control while allowing other users to access - // the generated artifacts. - int mkdir_result = mkdir(coverage_dir.c_str(), 0755); - - // If mkdir fails, it sets the global 'errno' variable to an error code - // indicating the reason. We check this code to specifically ignore the - // EEXIST error, which just means the directory already exists (this is not - // a real failure we need to warn about). - if (mkdir_result == 0 || errno == EEXIST) { - std::string graph_path = - absl::StrCat(coverage_dir, "/coverage_graph.txt"); - std::ofstream out(graph_path); - if (out.is_open()) { - out << dot_graph; - out.close(); - } else { - ABSL_LOG(WARNING) << "Failed to open file for writing: " << graph_path; - } - } else { - ABSL_LOG(WARNING) << "Failed to create directory: " << coverage_dir - << " (reason: " << strerror(errno) << ")"; - } - } - - cel::test::CoverageIndex& coverage_index_; -}; - class CelTest : public testing::Test { public: explicit CelTest(std::shared_ptr test_runner, @@ -345,7 +234,28 @@ int main(int argc, char** argv) { std::unique_ptr cel_test_context = std::move(cel_test_context_or.value()); + // We manually enable coverage here instead of just setting the + // `enable_coverage` flag on the context. This is intentional and necessary + // for this binary's reporting model. + // + // This binary needs a single coverage report for all tests run. + // We create `coverage_index` here, local to the `main` function, so its + // lifetime spans the entire test run. + // + // We must pass this specific instance to the + // `CoverageReportingEnvironment`, which Google Test calls after all + // dynamically registered tests are finished. + // + // If we just set the `enable_coverage` flag, the `TestRunner`'s + // constructor (as used in our `cc_test` files) would create its own + // internal `CoverageIndex`. That internal index would be destroyed + // with the `TestRunner` and would not populate the `coverage_index` + // instance needed by our global reporter. + // + // This manual approach ensures all tests populate the same `coverage_index` + // (the one local to `main`), which is then ready for the final report. cel::test::CoverageIndex coverage_index; + if (absl::GetFlag(FLAGS_collect_coverage)) { if (cel_test_context->runtime() != nullptr) { ABSL_CHECK_OK(cel::test::EnableCoverageInRuntime( @@ -378,7 +288,7 @@ int main(int argc, char** argv) { if (absl::GetFlag(FLAGS_collect_coverage)) { coverage_index.Init(*checked_expr); testing::AddGlobalTestEnvironment( - new CoverageReportingEnvironment(coverage_index)); + new cel::test::CoverageReportingEnvironment(coverage_index)); } return RUN_ALL_TESTS(); diff --git a/testing/testrunner/runner_lib.cc b/testing/testrunner/runner_lib.cc index ae09de255..28806cec7 100644 --- a/testing/testrunner/runner_lib.cc +++ b/testing/testrunner/runner_lib.cc @@ -23,6 +23,7 @@ #include "cel/expr/eval.pb.h" #include "absl/functional/overload.h" #include "absl/status/status.h" +#include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" @@ -41,6 +42,7 @@ #include "runtime/runtime.h" #include "testing/testrunner/cel_expression_source.h" #include "testing/testrunner/cel_test_context.h" +#include "testing/testrunner/coverage_index.h" #include "cel/expr/conformance/test/suite.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -397,12 +399,36 @@ absl::StatusOr TestRunner::GetCheckedExpr() const { source_ptr->source()); } +absl::Status TestRunner::EnableCoverage() { + if (test_context_ != nullptr && test_context_->enable_coverage()) { + coverage_index_ = std::make_unique(); + + if (test_context_->runtime() != nullptr) { + auto* runtime = const_cast(test_context_->runtime()); + CEL_RETURN_IF_ERROR(EnableCoverageInRuntime(*runtime, *coverage_index_)); + } else if (test_context_->cel_expression_builder() != nullptr) { + auto* builder = + const_cast( + test_context_->cel_expression_builder()); + CEL_RETURN_IF_ERROR( + EnableCoverageInCelExpressionBuilder(*builder, *coverage_index_)); + } + } + return absl::OkStatus(); +} + void TestRunner::RunTest(const TestCase& test_case) { // The arena has to be declared in RunTest because cel::Value returned by // EvalWithRuntime or EvalWithCelExpressionBuilder might contain pointers to // the arena. The arena has to be alive during the assertion. google::protobuf::Arena arena; + ASSERT_THAT(EnableCoverage(), absl_testing::IsOk()); ASSERT_OK_AND_ASSIGN(CheckedExpr checked_expr, GetCheckedExpr()); + + if (coverage_index_) { + coverage_index_->Init(checked_expr); + } + if (test_context_->runtime() != nullptr) { ASSERT_OK_AND_ASSIGN(cel::Value result, EvalWithRuntime(checked_expr, test_case, &arena)); diff --git a/testing/testrunner/runner_lib.h b/testing/testrunner/runner_lib.h index a04602d49..4f4fdfe40 100644 --- a/testing/testrunner/runner_lib.h +++ b/testing/testrunner/runner_lib.h @@ -16,11 +16,15 @@ #define THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_RUNNER_LIBRARY_H_ #include +#include #include +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "common/value.h" #include "testing/testrunner/cel_test_context.h" +#include "testing/testrunner/coverage_index.h" +#include "testing/testrunner/coverage_reporting.h" #include "cel/expr/conformance/test/suite.pb.h" #include "google/protobuf/arena.h" @@ -32,6 +36,14 @@ class TestRunner { explicit TestRunner(std::unique_ptr test_context) : test_context_(std::move(test_context)) {} + // Automatically reports coverage results. + ~TestRunner() { + if (coverage_index_) { + CoverageReportingEnvironment reporter(*coverage_index_); + reporter.TearDown(); + } + } + // Evaluates the checked expression in the test case, performs the // assertions against the expected result. void RunTest(const cel::expr::conformance::test::TestCase& test_case); @@ -39,6 +51,9 @@ class TestRunner { // Returns the checked expression for the test case. absl::StatusOr GetCheckedExpr() const; + // Returns the coverage report for the test case. + std::optional GetCoverageReport() const; + private: absl::StatusOr EvalWithRuntime( const cel::expr::CheckedExpr& checked_expr, @@ -61,7 +76,11 @@ class TestRunner { void AssertError(const cel::Value& computed, const cel::expr::conformance::test::TestOutput& output); + absl::Status EnableCoverage(); + std::unique_ptr test_context_; + + std::unique_ptr coverage_index_; }; } // namespace cel::test