diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD new file mode 100644 index 000000000..39087b598 --- /dev/null +++ b/testing/testrunner/BUILD @@ -0,0 +1,114 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +package( + default_testonly = True, + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) + +cc_library( + name = "cel_test_context", + hdrs = ["cel_test_context.h"], + deps = [ + "//compiler", + "//eval/public:cel_expression", + "//runtime", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/memory", + "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", + ], +) + +cc_library( + name = "runner_lib", + srcs = ["runner_lib.cc"], + hdrs = ["runner_lib.h"], + deps = [ + ":cel_test_context", + "//common:ast", + "//common:ast_proto", + "//common:value", + "//common/internal:value_conversion", + "//eval/public:activation", + "//eval/public:cel_expression", + "//eval/public:cel_value", + "//eval/public:transform_utility", + "//internal:status_macros", + "//internal:testing_no_main", + "//runtime", + "//runtime:activation", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", + "@com_google_protobuf//:differencer", + "@com_google_protobuf//:protobuf", + ], +) + +cc_library( + name = "cel_test_factories", + hdrs = ["cel_test_factories.h"], + deps = [ + ":cel_test_context", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status:statusor", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", + ], +) + +cc_test( + name = "runner_lib_test", + srcs = ["runner_lib_test.cc"], + deps = [ + ":cel_test_context", + ":runner_lib", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast_proto", + "//common:decl", + "//common:type", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", + "//eval/public:builtin_func_registrar", + "//eval/public:cel_expr_builder_factory", + "//eval/public:cel_expression", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//runtime", + "//runtime:runtime_builder", + "//runtime:standard_runtime_builder_factory", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + ], +) + +cc_library( + name = "runner", + srcs = ["runner_bin.cc"], + deps = [ + ":cel_test_context", + ":cel_test_factories", + ":runner_lib", + "//internal:testing_no_main", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//src/google/protobuf/io", + ], + alwayslink = True, +) diff --git a/testing/testrunner/cel_cc_test.bzl b/testing/testrunner/cel_cc_test.bzl new file mode 100644 index 000000000..07581ce3c --- /dev/null +++ b/testing/testrunner/cel_cc_test.bzl @@ -0,0 +1,70 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rules for triggering the cc impl of the CEL test runner.""" + +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +def cel_cc_test( + name, + test_suite = "", + filegroup = "", + deps = [], + test_data_path = "", + data = []): + """trigger the cc impl of the CEL test runner. + + This rule will generate a cc_test rule. This rule will be used to trigger + the cc impl of the cel_test rule. + + Args: + name: str name for the generated artifact + test_suite: str label of a file containing a test suite. The file should have a + .textproto extension. + filegroup: str label of a filegroup containing the test suite, the config and the checked + expression. + deps: list of dependencies for the cc_test rule. + data: list of data dependencies for the cc_test rule. + test_data_path: absolute path of the directory containing the test files. This is needed only + if the test files are not located in the same directory as the BUILD file. + """ + data, test_data_path = _update_data_with_test_files(data, filegroup, test_data_path, test_suite) + args = [] + + test_data_path = test_data_path.lstrip("/") + + if test_suite != "": + test_suite = test_data_path + "/" + test_suite + args.append("--test_suite_path=" + test_suite) + + cc_test( + name = name, + data = data, + args = args, + deps = ["//testing/testrunner:runner"] + deps, + ) + +def _update_data_with_test_files(data, filegroup, test_data_path, test_suite): + """Updates the data with the test files.""" + + if filegroup != "": + data = data + [filegroup] + elif test_data_path != "" and test_data_path != native.package_name(): + if test_suite != "": + data = data + [test_data_path + ":" + test_suite] + else: + test_data_path = native.package_name() + if test_suite != "": + data = data + [test_suite] + return data, test_data_path diff --git a/testing/testrunner/cel_test_context.h b/testing/testrunner/cel_test_context.h new file mode 100644 index 000000000..d2b0d841c --- /dev/null +++ b/testing/testrunner/cel_test_context.h @@ -0,0 +1,135 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TOOLS_TESTRUNNER_CEL_TEST_CONTEXT_H_ +#define THIRD_PARTY_CEL_CPP_TOOLS_TESTRUNNER_CEL_TEST_CONTEXT_H_ + +#include +#include +#include + +#include "cel/expr/checked.pb.h" +#include "absl/base/nullability.h" +#include "absl/memory/memory.h" +#include "compiler/compiler.h" +#include "eval/public/cel_expression.h" +#include "runtime/runtime.h" + +namespace cel::test { + +// Struct to hold optional parameters for `CelTestContext`. +struct CelTestContextOptions { + // The primary compiled CEL expression to be evaluated by the test. + std::optional checked_expr; + + // An optional CEL compiler. This is only required for test cases where + // input or output values are themselves CEL expressions that need to be + // resolved at runtime. + std::unique_ptr compiler = nullptr; +}; + +// The context class for a CEL test, holding configurations needed to evaluate +// compiled CEL expressions. +class CelTestContext { + public: + // Creates a CelTestContext using a `CelExpressionBuilder`. + // + // The `CelExpressionBuilder` helps in setting up the environment for + // building the CEL expression. + // + // Example usage: + // + // CEL_REGISTER_TEST_CONTEXT_FACTORY( + // []() -> absl::StatusOr> { + // // SAFE: This setup code now runs when the lambda is invoked at + // runtime, + // // long after all static initializations are complete. + // auto cel_expression_builder = + // google::api::expr::runtime::CreateCelExpressionBuilder(); + // CelTestContextOptions options; + // return CelTestContext::CreateFromCelExpressionBuilder( + // std::move(cel_expression_builder), std::move(options)); + // }); + static std::unique_ptr CreateFromCelExpressionBuilder( + std::unique_ptr + cel_expression_builder, + CelTestContextOptions options) { + return absl::WrapUnique(new CelTestContext( + std::move(cel_expression_builder), std::move(options))); + } + + // Creates a CelTestContext using a `cel::Runtime`. + // + // The `cel::Runtime` is used to evaluate the CEL expression by managing + // the state needed to generate Program. + static std::unique_ptr CreateFromRuntime( + std::unique_ptr runtime, + CelTestContextOptions options) { + return absl::WrapUnique( + new CelTestContext(std::move(runtime), std::move(options))); + } + + const cel::Runtime* absl_nullable runtime() const { return runtime_.get(); } + + const google::api::expr::runtime::CelExpressionBuilder* absl_nullable + cel_expression_builder() const { + return cel_expression_builder_.get(); + } + + const cel::Compiler* absl_nullable compiler() const { + return cel_test_context_options_.compiler.get(); + } + + std::optional checked_expr() const { + return cel_test_context_options_.checked_expr; + } + + private: + // Delete copy and move constructors. + CelTestContext(const CelTestContext&) = delete; + CelTestContext& operator=(const CelTestContext&) = delete; + CelTestContext(CelTestContext&&) = delete; + CelTestContext& operator=(CelTestContext&&) = delete; + + // Make the constructors private to enforce the use of the factory methods. + CelTestContext( + std::unique_ptr + cel_expression_builder, + CelTestContextOptions options) + : cel_test_context_options_(std::move(options)), + cel_expression_builder_(std::move(cel_expression_builder)) {} + + CelTestContext(std::unique_ptr runtime, + CelTestContextOptions options) + : cel_test_context_options_(std::move(options)), + runtime_(std::move(runtime)) {} + + // Configuration for the expression to be executed. + CelTestContextOptions cel_test_context_options_; + + // This helps in setting up the environment for building the CEL + // expression. Users should either provide a runtime, or the + // CelExpressionBuilder. + std::unique_ptr + cel_expression_builder_; + + // The runtime is used to evaluate the CEL expression by managing the state + // needed to generate Program. Users should either provide a runtime, or the + // CelExpressionBuilder. + std::unique_ptr runtime_; +}; + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TOOLS_TESTRUNNER_CEL_TEST_CONTEXT_H_ diff --git a/testing/testrunner/cel_test_factories.h b/testing/testrunner/cel_test_factories.h new file mode 100644 index 000000000..61058be13 --- /dev/null +++ b/testing/testrunner/cel_test_factories.h @@ -0,0 +1,91 @@ +// Copyright 2025 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_CEL_TEST_FACTORIES_H_ +#define THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_CEL_TEST_FACTORIES_H_ + +#include +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/log/absl_check.h" +#include "absl/status/statusor.h" +#include "testing/testrunner/cel_test_context.h" +#include "cel/expr/conformance/test/suite.pb.h" +namespace cel::test { +namespace internal { + +using CelTestContextFactoryFn = + std::function>()>; +using CelTestSuiteFactoryFn = + std::function; + +// Returns the factory function for creating a CelTestContext. +inline CelTestContextFactoryFn& GetCelTestContextFactory() { + static absl::NoDestructor factory; + return *factory; +} + +// Sets the factory function for creating a CelTestContext. Only one factory +// function can be set. Usage details can be found in cel_test_context.h. +inline bool SetCelTestContextFactory(CelTestContextFactoryFn factory) { + ABSL_DCHECK(GetCelTestContextFactory() == nullptr) + << "CelTestContextFactory is already set."; + GetCelTestContextFactory() = std::move(factory); + return true; +} + +// Returns the factory function for creating a CelTestSuite. +inline CelTestSuiteFactoryFn& GetCelTestSuiteFactory() { + static absl::NoDestructor factory; + return *factory; +} + +// Sets the factory function for creating a CelTestSuite. Only one factory +// function can be set. +inline bool SetCelTestSuiteFactory(CelTestSuiteFactoryFn factory) { + ABSL_DCHECK(GetCelTestSuiteFactory() == nullptr) + << "CelTestSuiteFactory is already set."; + GetCelTestSuiteFactory() = std::move(factory); + return true; +} +} // namespace internal + +// Register cel test context factories from a function or lambda. +// +// The return value of `factory_fn` should be a +// `absl::StatusOr>>`. +#define CEL_REGISTER_TEST_CONTEXT_FACTORY(factory_fn) \ + namespace { \ + const bool kTestContextFactoryRegistrationResult_##__LINE__ = \ + ::cel::test::internal::SetCelTestContextFactory(factory_fn); \ + } + +// Register cel test suite factory from a function or lambda. This is used to +// provide a custom test suite to the test runner which is useful for cases +// where the test suite is dynamically generated or where the test suite needs +// to be generated from a user provided source. +// +// The return value of `factory_fn` should be a +// `::cel::expr::conformance::test::TestSuite`. +#define CEL_REGISTER_TEST_SUITE_FACTORY(factory_fn) \ + namespace { \ + const bool kTestSuiteFactoryRegistrationResult_##__LINE__ = \ + ::cel::test::internal::SetCelTestSuiteFactory(factory_fn); \ + } + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_CEL_TEST_FACTORIES_H_ diff --git a/testing/testrunner/runner_bin.cc b/testing/testrunner/runner_bin.cc new file mode 100644 index 000000000..a307d45c0 --- /dev/null +++ b/testing/testrunner/runner_bin.cc @@ -0,0 +1,129 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This binary is a test runner for CEL tests. It is used to run CEL tests +// written in the CEL test suite format. +#include +#include +#include +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "internal/testing.h" +#include "testing/testrunner/cel_test_context.h" +#include "testing/testrunner/cel_test_factories.h" +#include "testing/testrunner/runner_lib.h" +#include "cel/expr/conformance/test/suite.pb.h" +#include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/text_format.h" + +ABSL_FLAG(std::string, test_suite_path, "", + "The path to the file containing the test suite to run."); + +namespace { + +using ::cel::expr::conformance::test::TestCase; +using ::cel::expr::conformance::test::TestSuite; +using ::cel::test::TestRunner; + +class CelTest : public testing::Test { + public: + explicit CelTest(std::shared_ptr test_runner, + const TestCase& test_case) + : test_runner_(std::move(test_runner)), test_case_(test_case) {} + + void TestBody() override { test_runner_->RunTest(test_case_); } + + private: + std::shared_ptr test_runner_; + TestCase test_case_; +}; + +absl::Status RegisterTests(const TestSuite& test_suite, + const std::shared_ptr& test_runner) { + for (const auto& section : test_suite.sections()) { + for (const TestCase& test_case : section.tests()) { + testing::RegisterTest( + test_suite.name().c_str(), + absl::StrCat(section.name(), "/", test_case.name()).c_str(), nullptr, + nullptr, __FILE__, __LINE__, [&test_runner, test_case]() -> CelTest* { + return new CelTest(test_runner, test_case); + }); + } + } + return absl::OkStatus(); +} + +TestSuite ReadTestSuiteFromPath(std::string_view test_suite_path) { + TestSuite test_suite; + { + std::ifstream in; + in.open(std::string(test_suite_path), + std::ios_base::in | std::ios_base::binary); + if (!in.is_open()) { + ABSL_LOG(FATAL) << "failed to open file: " << test_suite_path; + } + google::protobuf::io::IstreamInputStream stream(&in); + if (!google::protobuf::TextFormat::Parse(&stream, &test_suite)) { + ABSL_LOG(FATAL) << "failed to parse file: " << test_suite_path; + } + } + return test_suite; +} + +TestSuite GetTestSuite() { + std::string test_suite_path = absl::GetFlag(FLAGS_test_suite_path); + if (!test_suite_path.empty()) { + return ReadTestSuiteFromPath(test_suite_path); + } + + // If no test suite path is provided, use the factory function to get the + // test suite after checking if the factory function is empty or not. + std::function test_suite_factory = + cel::test::internal::GetCelTestSuiteFactory(); + if (test_suite_factory == nullptr) { + ABSL_LOG(FATAL) + << "No CEL test suite provided. Please provide a test suite using " + "either the bzl macro or the CEL_REGISTER_TEST_SUITE_FACTORY " + "preprocessor macro."; + } + return test_suite_factory(); +} +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + + // Create a test context using the factory function returned by the global + // factory function provider which was initialized by the user. + absl::StatusOr> cel_test_context = + cel::test::internal::GetCelTestContextFactory()(); + if (!cel_test_context.ok()) { + ABSL_LOG(FATAL) << "Failed to create CEL test context: " + << cel_test_context.status(); + } + auto test_runner = + std::make_shared(std::move(cel_test_context.value())); + ABSL_CHECK_OK(RegisterTests(GetTestSuite(), test_runner)); + + return RUN_ALL_TESTS(); +} diff --git a/testing/testrunner/runner_lib.cc b/testing/testrunner/runner_lib.cc new file mode 100644 index 000000000..e0fe723dd --- /dev/null +++ b/testing/testrunner/runner_lib.cc @@ -0,0 +1,280 @@ +// Copyright 2025 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "testing/testrunner/runner_lib.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "common/internal/value_conversion.h" +#include "common/value.h" +#include "eval/public/activation.h" +#include "eval/public/cel_expression.h" +#include "eval/public/cel_value.h" +#include "eval/public/transform_utility.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "runtime/activation.h" +#include "runtime/runtime.h" +#include "testing/testrunner/cel_test_context.h" +#include "cel/expr/conformance/test/suite.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" +#include "google/protobuf/util/field_comparator.h" +#include "google/protobuf/util/message_differencer.h" + +namespace cel::test { +namespace { + +using ::cel::expr::conformance::test::InputValue; +using ::cel::expr::conformance::test::TestCase; +using ::cel::expr::conformance::test::TestOutput; +using ::cel::expr::CheckedExpr; +using ::google::api::expr::runtime::CelExpression; +using ::google::api::expr::runtime::CelValue; +using ::google::api::expr::runtime::ValueToCelValue; +using ValueProto = ::cel::expr::Value; +using ::google::api::expr::runtime::Activation; + +absl::StatusOr> Plan( + const CheckedExpr& checked_expr, const cel::Runtime* runtime) { + std::unique_ptr ast; + CEL_ASSIGN_OR_RETURN(ast, cel::CreateAstFromCheckedExpr(checked_expr)); + if (ast == nullptr) { + return absl::InternalError("No expression provided for testing."); + } + return runtime->CreateProgram(std::move(ast)); +} + +const google::protobuf::DescriptorPool* GetDescriptorPool(const CelTestContext& context) { + return context.cel_expression_builder() != nullptr + ? google::protobuf::DescriptorPool::generated_pool() + : context.runtime()->GetDescriptorPool(); +} + +google::protobuf::MessageFactory* GetMessageFactory(const CelTestContext& context) { + return context.cel_expression_builder() != nullptr + ? google::protobuf::MessageFactory::generated_factory() + : context.runtime()->GetMessageFactory(); +} + +absl::StatusOr EvalWithModernBindings( + const CheckedExpr& checked_expr, const CelTestContext& context, + const cel::Activation& activation, google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + Plan(checked_expr, context.runtime())); + return program->Evaluate(arena, activation); +} + +absl::StatusOr EvalWithLegacyBindings( + const CheckedExpr& checked_expr, const CelTestContext& context, + const Activation& activation, google::protobuf::Arena* arena) { + const auto* builder = context.cel_expression_builder(); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr sub_expression, + builder->CreateExpression(&checked_expr)); + + CEL_ASSIGN_OR_RETURN(CelValue legacy_result, + sub_expression->Evaluate(activation, arena)); + + ValueProto result_proto; + CEL_RETURN_IF_ERROR(CelValueToValue(legacy_result, &result_proto)); + return FromExprValue(result_proto, GetDescriptorPool(context), + GetMessageFactory(context), arena); +} + +absl::StatusOr ResolveValue(const InputValue& input_value, + const CelTestContext& context, + google::protobuf::Arena* arena) { + return FromExprValue(input_value.value(), GetDescriptorPool(context), + GetMessageFactory(context), arena); +} + +absl::StatusOr ResolveExpr(absl::string_view expr, + const CelTestContext& context, + google::protobuf::Arena* arena) { + const auto* compiler = context.compiler(); + if (compiler == nullptr) { + return absl::InvalidArgumentError( + "Test case uses an expression but no compiler was provided."); + } + CEL_ASSIGN_OR_RETURN(auto validation_result, compiler->Compile(expr)); + if (!validation_result.IsValid()) { + return absl::InternalError(validation_result.FormatError()); + } + CheckedExpr checked_expr; + CEL_RETURN_IF_ERROR( + AstToCheckedExpr(*validation_result.GetAst(), &checked_expr)); + if (context.runtime() != nullptr) { + cel::Activation empty_activation; + return EvalWithModernBindings(checked_expr, context, empty_activation, + arena); + } else { + Activation empty_activation; + return EvalWithLegacyBindings(checked_expr, context, empty_activation, + arena); + } +} + +absl::StatusOr ResolveInputValue(const InputValue& input_value, + const CelTestContext& context, + google::protobuf::Arena* arena) { + switch (input_value.kind_case()) { + case InputValue::kValue: { + return ResolveValue(input_value, context, arena); + } + case InputValue::kExpr: { + return ResolveExpr(input_value.expr(), context, arena); + } + default: + return absl::InvalidArgumentError("Unknown InputValue kind."); + } +} + +absl::StatusOr CreateModernActivationFromBindings( + const TestCase& test_case, const CelTestContext& context, + google::protobuf::Arena* arena) { + cel::Activation activation; + for (const auto& binding : test_case.input()) { + CEL_ASSIGN_OR_RETURN( + Value value, + ResolveInputValue(/*input_value=*/binding.second, context, arena)); + activation.InsertOrAssignValue(/*name=*/binding.first, std::move(value)); + } + return activation; +} + +absl::StatusOr CreateLegacyActivationFromBindings( + const TestCase& test_case, const CelTestContext& context, + google::protobuf::Arena* arena) { + Activation activation; + auto* message_factory = GetMessageFactory(context); + auto* descriptor_pool = GetDescriptorPool(context); + for (const auto& binding : test_case.input()) { + CEL_ASSIGN_OR_RETURN( + cel::Value resolved_cel_value, + ResolveInputValue(/*input_value=*/binding.second, context, arena)); + CEL_ASSIGN_OR_RETURN(ValueProto value_proto, + ToExprValue(resolved_cel_value, descriptor_pool, + message_factory, arena)); + CEL_ASSIGN_OR_RETURN(CelValue value, ValueToCelValue(value_proto, arena)); + activation.InsertValue(/*name=*/binding.first, value); + } + return activation; +} + +bool IsEqual(const ValueProto& expected, const ValueProto& actual) { + static auto* kFieldComparator = []() { + auto* field_comparator = new google::protobuf::util::DefaultFieldComparator(); + field_comparator->set_treat_nan_as_equal(true); + return field_comparator; + }(); + static auto* kDifferencer = []() { + auto* differencer = new google::protobuf::util::MessageDifferencer(); + differencer->set_message_field_comparison( + google::protobuf::util::MessageDifferencer::EQUIVALENT); + differencer->set_field_comparator(kFieldComparator); + const auto* descriptor = cel::expr::MapValue::descriptor(); + const auto* entries_field = descriptor->FindFieldByName("entries"); + const auto* key_field = + entries_field->message_type()->FindFieldByName("key"); + differencer->TreatAsMap(entries_field, key_field); + return differencer; + }(); + return kDifferencer->Compare(expected, actual); +} + +MATCHER_P(MatchesValue, expected, "") { return IsEqual(arg, expected); } +} // namespace + +void TestRunner::AssertValue(const cel::Value& computed, + const TestOutput& output, google::protobuf::Arena* arena) { + ValueProto expected_value_proto; + const auto* descriptor_pool = GetDescriptorPool(*test_context_); + auto* message_factory = GetMessageFactory(*test_context_); + if (output.has_result_value()) { + expected_value_proto = output.result_value(); + } else if (output.has_result_expr()) { + InputValue input_value; + input_value.set_expr(output.result_expr()); + ASSERT_OK_AND_ASSIGN(cel::Value resolved_cel_value, + ResolveInputValue(input_value, *test_context_, arena)); + ASSERT_OK_AND_ASSIGN(expected_value_proto, + ToExprValue(resolved_cel_value, descriptor_pool, + message_factory, arena)); + } + ValueProto computed_expr_value; + ASSERT_OK_AND_ASSIGN( + computed_expr_value, + ToExprValue(computed, descriptor_pool, message_factory, arena)); + EXPECT_THAT(expected_value_proto, MatchesValue(computed_expr_value)); +} + +void TestRunner::Assert(const cel::Value& computed, const TestCase& test_case, + google::protobuf::Arena* arena) { + TestOutput output = test_case.output(); + if (output.has_result_value() || output.has_result_expr()) { + AssertValue(computed, output, arena); + } else if (output.has_eval_error()) { + ADD_FAILURE() << "Error assertion not implemented yet."; + } else if (output.has_unknown()) { + ADD_FAILURE() << "Unknown assertions not implemented yet."; + } else { + ADD_FAILURE() << "Unexpected output kind."; + } +} + +absl::StatusOr TestRunner::EvalWithRuntime( + const TestCase& test_case, google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN( + cel::Activation activation, + CreateModernActivationFromBindings(test_case, *test_context_, arena)); + return EvalWithModernBindings(*test_context_->checked_expr(), *test_context_, + activation, arena); +} + +absl::StatusOr TestRunner::EvalWithCelExpressionBuilder( + const TestCase& test_case, google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN( + Activation activation, + CreateLegacyActivationFromBindings(test_case, *test_context_, arena)); + return EvalWithLegacyBindings(*test_context_->checked_expr(), *test_context_, + activation, arena); +} + +void TestRunner::RunTest(const TestCase& test_case) { + // The arena has to be declared in RunTest because cel::Value returned by + // EvalWithRuntime or EvalWithCelExpressionBuilder might contain pointers to + // the arena. The arena has to be alive during the assertion. + google::protobuf::Arena arena; + const auto& checked_expr = test_context_->checked_expr(); + if (!checked_expr.has_value()) { + ADD_FAILURE() << "CheckedExpr is required for evaluation."; + return; + } + if (test_context_->runtime() != nullptr) { + ASSERT_OK_AND_ASSIGN(cel::Value result, EvalWithRuntime(test_case, &arena)); + ASSERT_NO_FATAL_FAILURE(Assert(result, test_case, &arena)); + } else if (test_context_->cel_expression_builder() != nullptr) { + ASSERT_OK_AND_ASSIGN(cel::Value result, + EvalWithCelExpressionBuilder(test_case, &arena)); + ASSERT_NO_FATAL_FAILURE(Assert(result, test_case, &arena)); + } +} +} // namespace cel::test diff --git a/testing/testrunner/runner_lib.h b/testing/testrunner/runner_lib.h new file mode 100644 index 000000000..895a07950 --- /dev/null +++ b/testing/testrunner/runner_lib.h @@ -0,0 +1,61 @@ +// Copyright 2025 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_RUNNER_LIBRARY_H_ +#define THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_RUNNER_LIBRARY_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "common/value.h" +#include "testing/testrunner/cel_test_context.h" +#include "cel/expr/conformance/test/suite.pb.h" +#include "google/protobuf/arena.h" + +namespace cel::test { + +// The test runner class for running CEL tests. +class TestRunner { + public: + explicit TestRunner(std::unique_ptr test_context) + : test_context_(std::move(test_context)) {} + + // Evaluates the checked expression in the test case, performs the + // assertions against the expected result. + void RunTest(const cel::expr::conformance::test::TestCase& test_case); + + private: + absl::StatusOr EvalWithRuntime( + const cel::expr::conformance::test::TestCase& test_case, + google::protobuf::Arena* arena); + + absl::StatusOr EvalWithCelExpressionBuilder( + const cel::expr::conformance::test::TestCase& test_case, + google::protobuf::Arena* arena); + + void Assert(const cel::Value& computed, + const cel::expr::conformance::test::TestCase& test_case, + google::protobuf::Arena* arena); + + void AssertValue(const cel::Value& computed, + const cel::expr::conformance::test::TestOutput& output, + google::protobuf::Arena* arena); + + std::unique_ptr test_context_; +}; + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_RUNNER_LIBRARY_H_ diff --git a/testing/testrunner/runner_lib_test.cc b/testing/testrunner/runner_lib_test.cc new file mode 100644 index 000000000..2499a0b5c --- /dev/null +++ b/testing/testrunner/runner_lib_test.cc @@ -0,0 +1,454 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "testing/testrunner/runner_lib.h" + +#include +#include +#include + +#include "gtest/gtest-spi.h" +#include "absl/log/absl_check.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast_proto.h" +#include "common/decl.h" +#include "common/type.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "eval/public/builtin_func_registrar.h" +#include "eval/public/cel_expr_builder_factory.h" +#include "eval/public/cel_expression.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "testing/testrunner/cel_test_context.h" +#include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" + +namespace cel::test { +namespace { + +using ::cel::expr::conformance::proto3::TestAllTypes; +using ::cel::expr::conformance::test::TestCase; +using ::cel::expr::CheckedExpr; + +template +T ParseTextProtoOrDie(absl::string_view text_proto) { + T result; + ABSL_CHECK(google::protobuf::TextFormat::ParseFromString(text_proto, &result)); + return result; +} + +absl::StatusOr> CreateBasicCompiler() { + CEL_ASSIGN_OR_RETURN( + std::unique_ptr builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCompilerLibrary())); + cel::TypeCheckerBuilder& checker_builder = builder->GetCheckerBuilder(); + CEL_RETURN_IF_ERROR( + checker_builder.AddVariable(cel::MakeVariableDecl("x", cel::IntType()))); + CEL_RETURN_IF_ERROR( + checker_builder.AddVariable(cel::MakeVariableDecl("y", cel::IntType()))); + return std::move(builder)->Build(); +} + +absl::StatusOr> CreateTestRuntime() { + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder standard_runtime_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetTestingDescriptorPool(), {})); + return std::move(standard_runtime_builder).Build(); +} + +absl::StatusOr< + std::unique_ptr> +CreateTestCelExpressionBuilder() { + auto builder = google::api::expr::runtime::CreateCelExpressionBuilder(); + CEL_RETURN_IF_ERROR(google::api::expr::runtime::RegisterBuiltinFunctions( + builder->GetRegistry())); + return builder; +} + +class TestRunnerTest : public ::testing::Test { + public: + void SetUp() override { + // Create a compiler. + ASSERT_OK_AND_ASSIGN(compiler_, CreateBasicCompiler()); + } + + protected: + std::unique_ptr compiler_; +}; + +TEST_F(TestRunnerTest, BasicTestWithRuntimeReportsSuccess) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("{'sum': x + y, 'literal': 3}")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a runtime. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 1 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { + result_value { + map_value { + entries { + key { string_value: "literal" } + value { int64_value: 3 } + } + entries { + key { string_value: "sum" } + value { int64_value: 3 } + } + } + } + } + )pb"); + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} + +TEST_F(TestRunnerTest, BasicTestWithRuntimeReportsFailure) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("x + y == 3")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a runtime. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 1 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { result_value { bool_value: false } } + )pb"); + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), + "bool_value: true"); // Expected true; Got false +} + +TEST_F(TestRunnerTest, BasicTestWithBuilderReportsSuccess) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("{'sum': x + y, 'literal': 3}")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a builder. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTestCelExpressionBuilder()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 1 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { + result_value { + map_value { + entries { + key { string_value: "literal" } + value { int64_value: 3 } + } + entries { + key { string_value: "sum" } + value { int64_value: 3 } + } + } + } + } + )pb"); + TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( + std::move(builder), /*options=*/{.checked_expr = checked_expr})); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} + +TEST_F(TestRunnerTest, BasicTestWithBuilderReportsFailure) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("x + y == 3")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a builder. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTestCelExpressionBuilder()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 1 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { result_value { bool_value: false } } + )pb"); + TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( + std::move(builder), /*options=*/{.checked_expr = checked_expr})); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), + "bool_value: true"); // Expected true; Got false +} + +TEST_F(TestRunnerTest, DynamicInputAndOutputWithRuntimeReportsSuccess) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a runtime. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { expr: "1 + 1" } + } + input { + key: "y" + value { expr: "10 - 7" } + } + output { result_expr: "7 - 2" } + )pb"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.checked_expr = checked_expr, + .compiler = std::move(compiler)})); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} + +TEST_F(TestRunnerTest, DynamicInputAndOutputWithRuntimeReportsFailure) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a runtime. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { expr: "1 + 1" } + } + input { + key: "y" + value { expr: "10 - 7" } + } + output { result_expr: "10" } + )pb"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.checked_expr = checked_expr, + .compiler = std::move(compiler)})); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), + "int64_value: 5"); // Expected 5; Got 10 +} + +TEST_F(TestRunnerTest, DynamicInputAndOutputWithBuilderReportsSuccess) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a CelExpressionBuilder. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTestCelExpressionBuilder()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { expr: "1 + 1" } + } + input { + key: "y" + value { expr: "10 - 7" } + } + output { result_expr: "7 - 2" } + )pb"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( + std::move(builder), + /*options=*/{.checked_expr = checked_expr, + .compiler = std::move(compiler)})); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} + +TEST_F(TestRunnerTest, DynamicInputAndOutputWithBuilderReportsFailure) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a CelExpressionBuilder. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTestCelExpressionBuilder()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { expr: "1 + 1" } + } + input { + key: "y" + value { expr: "10 - 7" } + } + output { result_expr: "10" } + )pb"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( + std::move(builder), + /*options=*/{.checked_expr = checked_expr, + .compiler = std::move(compiler)})); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), + "int64_value: 5"); // Expected 5; Got 10 +} + +TEST_F(TestRunnerTest, DynamicInputWithoutCompilerFails) { + const std::string expected_error = + "INVALID_ARGUMENT: Test case uses an expression but no compiler " + "was provided."; + + EXPECT_FATAL_FAILURE( + { + // Create a compiler. + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT( + cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { expr: "1 + 1" } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { result_value { int64_value: 3 } } + )pb"); + + // Create the expression builder. + ASSERT_OK_AND_ASSIGN(auto builder, CreateTestCelExpressionBuilder()); + + // Create the TestRunner without the compiler. + TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( + std::move(builder), + /*options=*/{.checked_expr = checked_expr})); + + test_runner.RunTest(test_case); + }, + expected_error); +} + +TEST(TestRunnerCustomCompilerTest, + RuntimeUsesRuntimePoolToResolveCustomProtoLiteral) { + // Create a custom CompilerBuilder. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + ASSERT_THAT(builder->AddLibrary(cel::StandardCompilerLibrary()), + absl_testing::IsOk()); + cel::TypeCheckerBuilder& checker_builder = builder->GetCheckerBuilder(); + ASSERT_THAT(checker_builder.AddVariable(cel::MakeVariableDecl( + "custom_var", cel::MessageType(TestAllTypes::descriptor()))), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(builder)->Build()); + + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("custom_var.single_int32 == 123")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + + // Create a runtime configured with the testing descriptor pool. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + + // Define the test case. The important part is the "custom_var" input, + // which forces 'ResolveValue' to run on a custom type. This succeeds because + // the testing descriptor pool (used by CreateTestRuntime()) is configured + // to contain the TestAllTypes descriptor. + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "custom_var" + value { + value { + object_value { + [type.googleapis.com/cel.expr.conformance.proto3.TestAllTypes] { + single_int32: 123 + } + } + } + } + } + output { result_value { bool_value: true } } + )pb"); + + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} +} // namespace +} // namespace cel::test