diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD index 523f275de..3b1e2f552 100644 --- a/testing/testrunner/BUILD +++ b/testing/testrunner/BUILD @@ -12,6 +12,7 @@ cc_library( name = "cel_test_context", hdrs = ["cel_test_context.h"], deps = [ + ":cel_expression_source", "//compiler", "//eval/public:cel_expression", "//runtime", @@ -26,7 +27,9 @@ cc_library( srcs = ["runner_lib.cc"], hdrs = ["runner_lib.h"], deps = [ + ":cel_expression_source", ":cel_test_context", + "//checker:validation_result", "//common:ast", "//common:ast_proto", "//common:value", @@ -39,8 +42,10 @@ cc_library( "//internal:testing_no_main", "//runtime", "//runtime:activation", + "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_cel_spec//proto/cel/expr:value_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", @@ -64,7 +69,14 @@ cc_library( cc_test( name = "runner_lib_test", srcs = ["runner_lib_test.cc"], + args = [ + "--test_cel_file_path=$(location //testing/testrunner/resources:test.cel)", + ], + data = [ + "//testing/testrunner/resources:test.cel", + ], deps = [ + ":cel_expression_source", ":cel_test_context", ":runner_lib", "//checker:type_checker_builder", @@ -84,6 +96,7 @@ cc_test( "//runtime", "//runtime:runtime_builder", "//runtime:standard_runtime_builder_factory", + "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", @@ -113,3 +126,9 @@ cc_library( ], alwayslink = True, ) + +cc_library( + name = "cel_expression_source", + hdrs = ["cel_expression_source.h"], + deps = ["@com_google_cel_spec//proto/cel/expr:checked_cc_proto"], +) diff --git a/testing/testrunner/cel_expression_source.h b/testing/testrunner/cel_expression_source.h new file mode 100644 index 000000000..dfdc61c5c --- /dev/null +++ b/testing/testrunner/cel_expression_source.h @@ -0,0 +1,81 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_CEL_EXPRESSION_SOURCE_H_ +#define THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_CEL_EXPRESSION_SOURCE_H_ + +#include +#include +#include + +#include "cel/expr/checked.pb.h" + +namespace cel::test { + +// A wrapper class that holds one of three possible sources for a CEL +// expression using a std::variant for type safety. +class CelExpressionSource { + public: + // Distinct wrapper types are used for string-based sources to disambiguate + // them within the std::variant. + struct RawExpression { + std::string value; + }; + + struct CelFile { + std::string path; + }; + + // The variant holds one of the three possible source types. + using SourceVariant = + std::variant; + + // Creates a CelExpressionSource from a compiled + // cel::expr::CheckedExpr. + static CelExpressionSource FromCheckedExpr( + cel::expr::CheckedExpr checked_expr) { + return CelExpressionSource(std::move(checked_expr)); + } + + // Creates a CelExpressionSource from a raw CEL expression string. + static CelExpressionSource FromRawExpression(std::string raw_expression) { + return CelExpressionSource(RawExpression{std::move(raw_expression)}); + } + + // Creates a CelExpressionSource from a file path pointing to a .cel file. + static CelExpressionSource FromCelFile(std::string cel_file_path) { + return CelExpressionSource(CelFile{std::move(cel_file_path)}); + } + + // Make copyable and movable. + CelExpressionSource(const CelExpressionSource&) = default; + CelExpressionSource& operator=(const CelExpressionSource&) = default; + CelExpressionSource(CelExpressionSource&&) = default; + CelExpressionSource& operator=(CelExpressionSource&&) = default; + + // Returns the underlying variant. The caller is expected to use std::visit + // to interact with the active value in a type-safe manner. + const SourceVariant& source() const { return source_; } + + private: + // A single private constructor enforces creation via the static factories. + explicit CelExpressionSource(SourceVariant source) + : source_(std::move(source)) {} + + // A single std::variant member efficiently stores one of the possible states. + SourceVariant source_; +}; +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTING_TESTRUNNER_CEL_EXPRESSION_SOURCE_H_ diff --git a/testing/testrunner/cel_test_context.h b/testing/testrunner/cel_test_context.h index d2b0d841c..903d2a8ea 100644 --- a/testing/testrunner/cel_test_context.h +++ b/testing/testrunner/cel_test_context.h @@ -25,17 +25,17 @@ #include "compiler/compiler.h" #include "eval/public/cel_expression.h" #include "runtime/runtime.h" - +#include "testing/testrunner/cel_expression_source.h" namespace cel::test { // Struct to hold optional parameters for `CelTestContext`. struct CelTestContextOptions { - // The primary compiled CEL expression to be evaluated by the test. - std::optional checked_expr; + // The source for the CEL expression to be evaluated in the test. + std::optional expression_source; - // An optional CEL compiler. This is only required for test cases where + // An optional CEL compiler. This is required for test cases where // input or output values are themselves CEL expressions that need to be - // resolved at runtime. + // resolved at runtime or cel expression source is raw string or cel file. std::unique_ptr compiler = nullptr; }; @@ -91,8 +91,10 @@ class CelTestContext { return cel_test_context_options_.compiler.get(); } - std::optional checked_expr() const { - return cel_test_context_options_.checked_expr; + const CelExpressionSource* absl_nullable expression_source() const { + return cel_test_context_options_.expression_source.has_value() + ? &cel_test_context_options_.expression_source.value() + : nullptr; } private: diff --git a/testing/testrunner/resources/BUILD b/testing/testrunner/resources/BUILD new file mode 100644 index 000000000..663f81780 --- /dev/null +++ b/testing/testrunner/resources/BUILD @@ -0,0 +1,12 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files( + ["test.cel"], +) + +filegroup( + name = "resources", + srcs = glob([ + "*.textproto", + ]), +) diff --git a/testing/testrunner/resources/simple_tests.textproto b/testing/testrunner/resources/simple_tests.textproto new file mode 100644 index 000000000..46275312f --- /dev/null +++ b/testing/testrunner/resources/simple_tests.textproto @@ -0,0 +1,62 @@ +# proto-file: google3/third_party/cel/spec/proto/cel/expr/conformance/test/suite.proto +# proto-message: cel.expr.conformance.test.TestSuite + +name: "simple_tests" +description: "Simple tests to validate the test runner." +sections: { + name: "simple_map_operations" + description: "Tests for map operations." + tests: { + name: "literal_and_sum" + description: "Test that a map can be created and values can be accessed." + input: { + key: "x" + value { value { int64_value: 1 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { + result_value { + map_value { + entries { + key { string_value: "literal" } + value { int64_value: 3 } + } + entries { + key { string_value: "sum" } + value { int64_value: 3 } + } + } + } + } + } + tests: { + name: "literal_and_sum_2_5" + description: "Test that a map can be created and values can be accessed." + input: { + key: "x" + value { value { int64_value: 2 } } + } + input { + key: "y" + value { value { int64_value: 5 } } + } + output { + result_value { + map_value { + entries { + key { string_value: "literal" } + value { int64_value: 3 } + } + entries { + key { string_value: "sum" } + value { int64_value: 7 } + } + } + } + } + } +} + diff --git a/testing/testrunner/resources/test.cel b/testing/testrunner/resources/test.cel new file mode 100644 index 000000000..e2a8707df --- /dev/null +++ b/testing/testrunner/resources/test.cel @@ -0,0 +1 @@ +x-y \ No newline at end of file diff --git a/testing/testrunner/runner_lib.cc b/testing/testrunner/runner_lib.cc index 26aa09ddf..00232c9c8 100644 --- a/testing/testrunner/runner_lib.cc +++ b/testing/testrunner/runner_lib.cc @@ -13,13 +13,20 @@ // limitations under the License. #include "testing/testrunner/runner_lib.h" +#include #include +#include +#include #include +#include #include "cel/expr/eval.pb.h" +#include "absl/functional/overload.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "checker/validation_result.h" #include "common/ast.h" #include "common/ast_proto.h" #include "common/internal/value_conversion.h" @@ -32,6 +39,7 @@ #include "internal/testing.h" #include "runtime/activation.h" #include "runtime/runtime.h" +#include "testing/testrunner/cel_expression_source.h" #include "testing/testrunner/cel_test_context.h" #include "cel/expr/conformance/test/suite.pb.h" #include "google/protobuf/arena.h" @@ -53,6 +61,38 @@ using ::google::api::expr::runtime::ValueToCelValue; using ValueProto = ::cel::expr::Value; using ::google::api::expr::runtime::Activation; +absl::StatusOr ReadFileToString(absl::string_view file_path) { + std::ifstream file_stream{std::string(file_path)}; + if (!file_stream.is_open()) { + return absl::NotFoundError( + absl::StrCat("Unable to open file: ", file_path)); + } + std::stringstream buffer; + buffer << file_stream.rdbuf(); + return buffer.str(); +} + +absl::StatusOr Compile(absl::string_view expression, + const CelTestContext& context) { + const auto* compiler = context.compiler(); + if (compiler == nullptr) { + return absl::InvalidArgumentError( + "A compiler must be provided to compile a raw expression or .cel " + "file."); + } + + CEL_ASSIGN_OR_RETURN(ValidationResult validation_result, + compiler->Compile(expression)); + if (!validation_result.IsValid()) { + return absl::InternalError(validation_result.FormatError()); + } + + CheckedExpr checked_expr; + CEL_RETURN_IF_ERROR( + AstToCheckedExpr(*validation_result.GetAst(), &checked_expr)); + return checked_expr; +} + absl::StatusOr> Plan( const CheckedExpr& checked_expr, const cel::Runtime* runtime) { std::unique_ptr ast; @@ -110,18 +150,7 @@ absl::StatusOr ResolveValue(const InputValue& input_value, absl::StatusOr ResolveExpr(absl::string_view expr, const CelTestContext& context, google::protobuf::Arena* arena) { - const auto* compiler = context.compiler(); - if (compiler == nullptr) { - return absl::InvalidArgumentError( - "Test case uses an expression but no compiler was provided."); - } - CEL_ASSIGN_OR_RETURN(auto validation_result, compiler->Compile(expr)); - if (!validation_result.IsValid()) { - return absl::InternalError(validation_result.FormatError()); - } - CheckedExpr checked_expr; - CEL_RETURN_IF_ERROR( - AstToCheckedExpr(*validation_result.GetAst(), &checked_expr)); + CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, Compile(expr, context)); if (context.runtime() != nullptr) { cel::Activation empty_activation; return EvalWithModernBindings(checked_expr, context, empty_activation, @@ -258,21 +287,44 @@ void TestRunner::Assert(const cel::Value& computed, const TestCase& test_case, } absl::StatusOr TestRunner::EvalWithRuntime( - const TestCase& test_case, google::protobuf::Arena* arena) { + const CheckedExpr& checked_expr, const TestCase& test_case, + google::protobuf::Arena* arena) { CEL_ASSIGN_OR_RETURN( cel::Activation activation, CreateModernActivationFromBindings(test_case, *test_context_, arena)); - return EvalWithModernBindings(*test_context_->checked_expr(), *test_context_, - activation, arena); + return EvalWithModernBindings(checked_expr, *test_context_, activation, + arena); } absl::StatusOr TestRunner::EvalWithCelExpressionBuilder( - const TestCase& test_case, google::protobuf::Arena* arena) { + const CheckedExpr& checked_expr, const TestCase& test_case, + google::protobuf::Arena* arena) { CEL_ASSIGN_OR_RETURN( Activation activation, CreateLegacyActivationFromBindings(test_case, *test_context_, arena)); - return EvalWithLegacyBindings(*test_context_->checked_expr(), *test_context_, - activation, arena); + return EvalWithLegacyBindings(checked_expr, *test_context_, activation, + arena); +} + +absl::StatusOr TestRunner::GetCheckedExpr() const { + const CelExpressionSource* source_ptr = test_context_->expression_source(); + if (source_ptr == nullptr) { + return absl::InvalidArgumentError("No expression source provided."); + } + return std::visit( + absl::Overload([](const cel::expr::CheckedExpr& v) + -> absl::StatusOr { return v; }, + [this](const CelExpressionSource::RawExpression& v) + -> absl::StatusOr { + return Compile(v.value, *test_context_); + }, + [this](const CelExpressionSource::CelFile& v) + -> absl::StatusOr { + CEL_ASSIGN_OR_RETURN(std::string contents, + ReadFileToString(v.path)); + return Compile(contents, *test_context_); + }), + source_ptr->source()); } void TestRunner::RunTest(const TestCase& test_case) { @@ -280,17 +332,15 @@ void TestRunner::RunTest(const TestCase& test_case) { // EvalWithRuntime or EvalWithCelExpressionBuilder might contain pointers to // the arena. The arena has to be alive during the assertion. google::protobuf::Arena arena; - const auto& checked_expr = test_context_->checked_expr(); - if (!checked_expr.has_value()) { - ADD_FAILURE() << "CheckedExpr is required for evaluation."; - return; - } + ASSERT_OK_AND_ASSIGN(CheckedExpr checked_expr, GetCheckedExpr()); if (test_context_->runtime() != nullptr) { - ASSERT_OK_AND_ASSIGN(cel::Value result, EvalWithRuntime(test_case, &arena)); + ASSERT_OK_AND_ASSIGN(cel::Value result, + EvalWithRuntime(checked_expr, test_case, &arena)); ASSERT_NO_FATAL_FAILURE(Assert(result, test_case, &arena)); } else if (test_context_->cel_expression_builder() != nullptr) { - ASSERT_OK_AND_ASSIGN(cel::Value result, - EvalWithCelExpressionBuilder(test_case, &arena)); + ASSERT_OK_AND_ASSIGN( + cel::Value result, + EvalWithCelExpressionBuilder(checked_expr, test_case, &arena)); ASSERT_NO_FATAL_FAILURE(Assert(result, test_case, &arena)); } } diff --git a/testing/testrunner/runner_lib.h b/testing/testrunner/runner_lib.h index be8736da4..40a5c50f4 100644 --- a/testing/testrunner/runner_lib.h +++ b/testing/testrunner/runner_lib.h @@ -38,10 +38,12 @@ class TestRunner { private: absl::StatusOr EvalWithRuntime( + const cel::expr::CheckedExpr& checked_expr, const cel::expr::conformance::test::TestCase& test_case, google::protobuf::Arena* arena); absl::StatusOr EvalWithCelExpressionBuilder( + const cel::expr::CheckedExpr& checked_expr, const cel::expr::conformance::test::TestCase& test_case, google::protobuf::Arena* arena); @@ -53,6 +55,8 @@ class TestRunner { const cel::expr::conformance::test::TestOutput& output, google::protobuf::Arena* arena); + absl::StatusOr GetCheckedExpr() const; + void AssertError(const cel::Value& computed, const cel::expr::conformance::test::TestOutput& output); diff --git a/testing/testrunner/runner_lib_test.cc b/testing/testrunner/runner_lib_test.cc index f67c44242..c95097f21 100644 --- a/testing/testrunner/runner_lib_test.cc +++ b/testing/testrunner/runner_lib_test.cc @@ -18,6 +18,7 @@ #include #include "gtest/gtest-spi.h" +#include "absl/flags/flag.h" #include "absl/log/absl_check.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" @@ -39,18 +40,23 @@ #include "runtime/runtime.h" #include "runtime/runtime_builder.h" #include "runtime/standard_runtime_builder_factory.h" +#include "testing/testrunner/cel_expression_source.h" #include "testing/testrunner/cel_test_context.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" +ABSL_FLAG(std::string, test_cel_file_path, "", + "Path to the .cel file for testing"); + namespace cel::test { namespace { using ::cel::expr::conformance::proto3::TestAllTypes; using ::cel::expr::conformance::test::TestCase; using ::cel::expr::CheckedExpr; +using ::google::api::expr::runtime::CelExpressionBuilder; template T ParseTextProtoOrDie(absl::string_view text_proto) { @@ -79,8 +85,7 @@ absl::StatusOr> CreateTestRuntime() { return std::move(standard_runtime_builder).Build(); } -absl::StatusOr< - std::unique_ptr> +absl::StatusOr> CreateTestCelExpressionBuilder() { auto builder = google::api::expr::runtime::CreateCelExpressionBuilder(); CEL_RETURN_IF_ERROR(google::api::expr::runtime::RegisterBuiltinFunctions( @@ -88,27 +93,47 @@ CreateTestCelExpressionBuilder() { return builder; } -class TestRunnerTest : public ::testing::Test { - public: - void SetUp() override { - // Create a compiler. - ASSERT_OK_AND_ASSIGN(compiler_, CreateBasicCompiler()); - } +// Creates a static, singleton instance of the basic compiler to be shared +// across tests, avoiding repeated setup costs. +const cel::Compiler& DefaultCompiler() { + static const cel::Compiler* instance = []() { + absl::StatusOr> s = CreateBasicCompiler(); + ABSL_QCHECK_OK(s.status()); + return s->release(); + }(); + return *instance; +} +enum class RuntimeApi { kRuntime, kBuilder }; + +// Parameterized test fixture for tests that are run against both the Runtime +// and the CelExpressionBuilder backends. +class TestRunnerParamTest : public ::testing::TestWithParam { protected: - std::unique_ptr compiler_; + // Helper to create the appropriate CelTestContext based on the test + // parameter. + absl::StatusOr> CreateTestContext( + CelTestContextOptions options) { + if (GetParam() == RuntimeApi::kRuntime) { + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + CreateTestRuntime()); + return CelTestContext::CreateFromRuntime(std::move(runtime), + std::move(options)); + } + CEL_ASSIGN_OR_RETURN(std::unique_ptr builder, + CreateTestCelExpressionBuilder()); + return CelTestContext::CreateFromCelExpressionBuilder(std::move(builder), + std::move(options)); + } }; -TEST_F(TestRunnerTest, BasicTestWithRuntimeReportsSuccess) { - // Compile the expression. - ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("{'sum': x + y, 'literal': 3}")); +TEST_P(TestRunnerParamTest, BasicTestReportsSuccess) { + ASSERT_OK_AND_ASSIGN( + cel::ValidationResult validation_result, + DefaultCompiler().Compile("{'sum': x + y, 'literal': 3}")); CheckedExpr checked_expr; ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), absl_testing::IsOk()); - // Create a runtime. - ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, - CreateTestRuntime()); TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" @@ -133,21 +158,21 @@ TEST_F(TestRunnerTest, BasicTestWithRuntimeReportsSuccess) { } } )pb"); - TestRunner test_runner(CelTestContext::CreateFromRuntime( - std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + ASSERT_OK_AND_ASSIGN( + auto context, CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr( + std::move(checked_expr))})); + TestRunner test_runner(std::move(context)); EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } -TEST_F(TestRunnerTest, BasicTestWithRuntimeReportsFailure) { - // Compile the expression. +TEST_P(TestRunnerParamTest, BasicTestReportsFailure) { ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("x + y == 3")); + DefaultCompiler().Compile("x + y == 3")); CheckedExpr checked_expr; ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), absl_testing::IsOk()); - // Create a runtime. - ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, - CreateTestRuntime()); TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" @@ -159,208 +184,183 @@ TEST_F(TestRunnerTest, BasicTestWithRuntimeReportsFailure) { } output { result_value { bool_value: false } } )pb"); - TestRunner test_runner(CelTestContext::CreateFromRuntime( - std::move(runtime), /*options=*/{.checked_expr = checked_expr})); - EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), - "bool_value: true"); // Expected true; Got false + ASSERT_OK_AND_ASSIGN( + auto context, CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr( + std::move(checked_expr))})); + TestRunner test_runner(std::move(context)); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), "bool_value: true"); } -TEST_F(TestRunnerTest, BasicTestWithBuilderReportsSuccess) { - // Compile the expression. +TEST_P(TestRunnerParamTest, DynamicInputAndOutputReportsSuccess) { ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("{'sum': x + y, 'literal': 3}")); + DefaultCompiler().Compile("x + y")); CheckedExpr checked_expr; ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), absl_testing::IsOk()); - // Create a builder. - ASSERT_OK_AND_ASSIGN( - std::unique_ptr builder, - CreateTestCelExpressionBuilder()); TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" - value { value { int64_value: 1 } } + value { expr: "1 + 1" } } input { key: "y" - value { value { int64_value: 2 } } - } - output { - result_value { - map_value { - entries { - key { string_value: "literal" } - value { int64_value: 3 } - } - entries { - key { string_value: "sum" } - value { int64_value: 3 } - } - } - } + value { expr: "10 - 7" } } + output { result_expr: "7 - 2" } )pb"); - TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( - std::move(builder), /*options=*/{.checked_expr = checked_expr})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + ASSERT_OK_AND_ASSIGN( + auto context, CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr( + std::move(checked_expr)), + .compiler = std::move(compiler)})); + TestRunner test_runner(std::move(context)); EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } -TEST_F(TestRunnerTest, BasicTestWithBuilderReportsFailure) { - // Compile the expression. +TEST_P(TestRunnerParamTest, DynamicInputAndOutputReportsFailure) { ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("x + y == 3")); + DefaultCompiler().Compile("x + y")); CheckedExpr checked_expr; ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), absl_testing::IsOk()); - // Create a builder. - ASSERT_OK_AND_ASSIGN( - std::unique_ptr builder, - CreateTestCelExpressionBuilder()); TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" - value { value { int64_value: 1 } } + value { expr: "1 + 1" } } input { key: "y" - value { value { int64_value: 2 } } + value { expr: "10 - 7" } } - output { result_value { bool_value: false } } + output { result_expr: "10" } )pb"); - TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( - std::move(builder), /*options=*/{.checked_expr = checked_expr})); - EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), - "bool_value: true"); // Expected true; Got false + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + ASSERT_OK_AND_ASSIGN( + auto context, CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr( + std::move(checked_expr)), + .compiler = std::move(compiler)})); + TestRunner test_runner(std::move(context)); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), "int64_value: 5"); } -TEST_F(TestRunnerTest, DynamicInputAndOutputWithRuntimeReportsSuccess) { - // Compile the expression. - ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("x + y")); - CheckedExpr checked_expr; - ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), - absl_testing::IsOk()); - // Create a runtime. - ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, - CreateTestRuntime()); +TEST_P(TestRunnerParamTest, RawExpressionWithCompilerReportsSuccess) { TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" - value { expr: "1 + 1" } + value { value { int64_value: 10 } } } input { key: "y" - value { expr: "10 - 7" } + value { value { int64_value: 3 } } } - output { result_expr: "7 - 2" } + output { result_value { int64_value: 7 } } )pb"); ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, CreateBasicCompiler()); - TestRunner test_runner(CelTestContext::CreateFromRuntime( - std::move(runtime), - /*options=*/{.checked_expr = checked_expr, - .compiler = std::move(compiler)})); + ASSERT_OK_AND_ASSIGN( + auto context, + CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromRawExpression("x - y"), + .compiler = std::move(compiler)})); + TestRunner test_runner(std::move(context)); EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } -TEST_F(TestRunnerTest, DynamicInputAndOutputWithRuntimeReportsFailure) { - // Compile the expression. - ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("x + y")); - CheckedExpr checked_expr; - ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), - absl_testing::IsOk()); - // Create a runtime. - ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, - CreateTestRuntime()); +TEST_P(TestRunnerParamTest, RawExpressionWithCompilerReportsFailure) { TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" - value { expr: "1 + 1" } + value { value { int64_value: 10 } } } input { key: "y" - value { expr: "10 - 7" } + value { value { int64_value: 3 } } } - output { result_expr: "10" } + output { result_value { int64_value: 100 } } )pb"); ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, CreateBasicCompiler()); - TestRunner test_runner(CelTestContext::CreateFromRuntime( - std::move(runtime), - /*options=*/{.checked_expr = checked_expr, - .compiler = std::move(compiler)})); - EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), - "int64_value: 5"); // Expected 5; Got 10 + ASSERT_OK_AND_ASSIGN( + auto context, + CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromRawExpression("x - y"), + .compiler = std::move(compiler)})); + TestRunner test_runner(std::move(context)); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), "int64_value: 7"); } -TEST_F(TestRunnerTest, DynamicInputAndOutputWithBuilderReportsSuccess) { - // Compile the expression. - ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("x + y")); - CheckedExpr checked_expr; - ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), - absl_testing::IsOk()); - // Create a CelExpressionBuilder. - ASSERT_OK_AND_ASSIGN( - std::unique_ptr builder, - CreateTestCelExpressionBuilder()); +TEST_P(TestRunnerParamTest, CelFileWithCompilerReportsSuccess) { + const std::string cel_file_path = absl::GetFlag(FLAGS_test_cel_file_path); + ASSERT_FALSE(cel_file_path.empty()) + << "Flag --test_cel_file_path must be set"; TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" - value { expr: "1 + 1" } + value { value { int64_value: 10 } } } input { key: "y" - value { expr: "10 - 7" } + value { value { int64_value: 3 } } } - output { result_expr: "7 - 2" } + output { result_value { int64_value: 7 } } )pb"); ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, CreateBasicCompiler()); - TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( - std::move(builder), - /*options=*/{.checked_expr = checked_expr, - .compiler = std::move(compiler)})); + ASSERT_OK_AND_ASSIGN( + auto context, + CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromCelFile(cel_file_path), + .compiler = std::move(compiler)})); + TestRunner test_runner(std::move(context)); EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } -TEST_F(TestRunnerTest, DynamicInputAndOutputWithBuilderReportsFailure) { - // Compile the expression. - ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("x + y")); - CheckedExpr checked_expr; - ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), - absl_testing::IsOk()); - // Create a CelExpressionBuilder. - ASSERT_OK_AND_ASSIGN( - std::unique_ptr builder, - CreateTestCelExpressionBuilder()); +TEST_P(TestRunnerParamTest, CelFileWithCompilerReportsFailure) { + const std::string cel_file_path = absl::GetFlag(FLAGS_test_cel_file_path); + ASSERT_FALSE(cel_file_path.empty()) + << "Flag --test_cel_file_path must be set"; TestCase test_case = ParseTextProtoOrDie(R"pb( input { key: "x" - value { expr: "1 + 1" } + value { value { int64_value: 10 } } } input { key: "y" - value { expr: "10 - 7" } + value { value { int64_value: 3 } } } - output { result_expr: "10" } + output { result_value { int64_value: 123 } } )pb"); ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, CreateBasicCompiler()); - TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( - std::move(builder), - /*options=*/{.checked_expr = checked_expr, - .compiler = std::move(compiler)})); - EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), - "int64_value: 5"); // Expected 5; Got 10 + ASSERT_OK_AND_ASSIGN( + auto context, + CreateTestContext( + /*options=*/{.expression_source = + CelExpressionSource::FromCelFile(cel_file_path), + .compiler = std::move(compiler)})); + TestRunner test_runner(std::move(context)); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), "int64_value: 7"); } -TEST_F(TestRunnerTest, DynamicInputWithoutCompilerFails) { +INSTANTIATE_TEST_SUITE_P(TestRunnerTests, TestRunnerParamTest, + ::testing::Values(RuntimeApi::kRuntime, + RuntimeApi::kBuilder)); + +TEST(TestRunnerStandaloneTest, DynamicInputWithoutCompilerFails) { const std::string expected_error = - "INVALID_ARGUMENT: Test case uses an expression but no compiler " - "was provided."; + "INVALID_ARGUMENT: A compiler must be provided to compile a raw " + "expression or .cel file."; EXPECT_FATAL_FAILURE( { @@ -392,15 +392,17 @@ TEST_F(TestRunnerTest, DynamicInputWithoutCompilerFails) { // Create the TestRunner without the compiler. TestRunner test_runner(CelTestContext::CreateFromCelExpressionBuilder( - std::move(builder), - /*options=*/{.checked_expr = checked_expr})); + /*cel_expression_builder=*/std::move(builder), + /*options=*/{.expression_source = + CelExpressionSource::FromCheckedExpr( + std::move(checked_expr))})); test_runner.RunTest(test_case); }, expected_error); } -TEST(TestRunnerCustomCompilerTest, +TEST(TestRunnerStandaloneTest, RuntimeUsesRuntimePoolToResolveCustomProtoLiteral) { // Create a custom CompilerBuilder. ASSERT_OK_AND_ASSIGN( @@ -447,14 +449,48 @@ TEST(TestRunnerCustomCompilerTest, )pb"); TestRunner test_runner(CelTestContext::CreateFromRuntime( - std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + std::move(runtime), + /*options=*/{.expression_source = CelExpressionSource::FromCheckedExpr( + std::move(checked_expr))})); EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } -TEST_F(TestRunnerTest, BasicTestWithErrorAssertion) { +TEST(TestRunnerStandaloneTest, RunTestFailsWhenNoExpressionSourceIsProvided) { + const std::string expected_error = + "INVALID_ARGUMENT: No expression source provided."; + + EXPECT_FATAL_FAILURE( + { + // Create a runtime. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 10 } } + } + input { + key: "y" + value { value { int64_value: 3 } } + } + output { result_value { int64_value: 123 } } + )pb"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBasicCompiler()); + + // Create a TestRunner but without an expression source. + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.compiler = std::move(compiler)})); + test_runner.RunTest(test_case); + }, + expected_error); +} + +TEST(TestRunnerStandaloneTest, BasicTestWithErrorAssertion) { // Compile the expression. ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("x + y")); + DefaultCompiler().Compile("x + y")); CheckedExpr checked_expr; ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), absl_testing::IsOk()); @@ -473,14 +509,16 @@ TEST_F(TestRunnerTest, BasicTestWithErrorAssertion) { } )pb"); TestRunner test_runner(CelTestContext::CreateFromRuntime( - std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + std::move(runtime), + /*options=*/{.expression_source = CelExpressionSource::FromCheckedExpr( + std::move(checked_expr))})); EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } -TEST_F(TestRunnerTest, BasicTestFailsWhenExpectingErrorButGotValue) { +TEST(TestRunnerStandaloneTest, BasicTestFailsWhenExpectingErrorButGotValue) { // Compile the expression. ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, - compiler_->Compile("1 + 1")); + DefaultCompiler().Compile("1 + 1")); CheckedExpr checked_expr; ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), absl_testing::IsOk()); @@ -495,7 +533,9 @@ TEST_F(TestRunnerTest, BasicTestFailsWhenExpectingErrorButGotValue) { } )pb"); TestRunner test_runner(CelTestContext::CreateFromRuntime( - std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + std::move(runtime), + /*options=*/{.expression_source = CelExpressionSource::FromCheckedExpr( + std::move(checked_expr))})); EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), "Expected error but got value"); } diff --git a/testing/testrunner/user_tests/BUILD b/testing/testrunner/user_tests/BUILD new file mode 100644 index 000000000..235c8441f --- /dev/null +++ b/testing/testrunner/user_tests/BUILD @@ -0,0 +1,54 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//testing/testrunner:cel_cc_test.bzl", "cel_cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "simple_user_test", + testonly = True, + srcs = ["simple.cc"], + deps = [ + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast_proto", + "//common:decl", + "//common:type", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", + "//internal:status_macros", + "//internal:testing_descriptor_pool", + "//runtime", + "//runtime:runtime_builder", + "//runtime:standard_runtime_builder_factory", + "//testing/testrunner:cel_expression_source", + "//testing/testrunner:cel_test_context", + "//testing/testrunner:cel_test_factories", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", + "@com_google_protobuf//:protobuf", + ], + alwayslink = True, +) + +cel_cc_test( + name = "simple_test", + filegroup = "//testing/testrunner/resources", + test_data_path = "//testing/testrunner/resources", + test_suite = "simple_tests.textproto", + deps = [ + ":simple_user_test", + ], +) + +cel_cc_test( + name = "simple_test_with_custom_test_suite", + filegroup = "//testing/testrunner/resources", + test_data_path = "//testing/testrunner/resources", + deps = [ + ":simple_user_test", + ], +) diff --git a/testing/testrunner/user_tests/simple.cc b/testing/testrunner/user_tests/simple.cc new file mode 100644 index 000000000..e199f6d17 --- /dev/null +++ b/testing/testrunner/user_tests/simple.cc @@ -0,0 +1,127 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "cel/expr/checked.pb.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast_proto.h" +#include "common/decl.h" +#include "common/type.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "internal/status_macros.h" +#include "internal/testing_descriptor_pool.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "testing/testrunner/cel_expression_source.h" +#include "testing/testrunner/cel_test_context.h" +#include "testing/testrunner/cel_test_factories.h" +#include "google/protobuf/text_format.h" + +namespace cel::testing { + +using ::cel::test::CelTestContext; +using ::cel::expr::CheckedExpr; + +template +T ParseTextProtoOrDie(absl::string_view text_proto) { + T result; + ABSL_CHECK(google::protobuf::TextFormat::ParseFromString(text_proto, &result)); + return result; +} + +CEL_REGISTER_TEST_SUITE_FACTORY([]() { + return ParseTextProtoOrDie(R"pb( + name: "custom_test_suite_tests" + description: "Simple tests to validate the test runner." + sections: { + name: "simple_map_operations" + description: "Tests for map operations." + tests: { + name: "literal_and_sum" + description: "Test that a map can be created and values can be accessed." + input: { + key: "x" + value { value { int64_value: 1 } } + } + input { + key: "y" + value { value { int64_value: 2 } } + } + output { + result_value { + map_value { + entries { + key { string_value: "literal" } + value { int64_value: 3 } + } + entries { + key { string_value: "sum" } + value { int64_value: 3 } + } + } + } + } + } + } + )pb"); +}); + +CEL_REGISTER_TEST_CONTEXT_FACTORY( + []() -> absl::StatusOr> { + ABSL_LOG(INFO) << "Creating test context"; + + // Create a compiler. + CEL_ASSIGN_OR_RETURN( + std::unique_ptr builder, + cel::NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCompilerLibrary())); + cel::TypeCheckerBuilder& checker_builder = builder->GetCheckerBuilder(); + CEL_RETURN_IF_ERROR(checker_builder.AddVariable( + cel::MakeVariableDecl("x", cel::IntType()))); + CEL_RETURN_IF_ERROR(checker_builder.AddVariable( + cel::MakeVariableDecl("y", cel::IntType()))); + CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + builder->Build()); + + // Compile the expression. + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile("{'sum': x + y, 'literal': 3}")); + CheckedExpr checked_expr; + CEL_RETURN_IF_ERROR( + cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr)); + + // Create a runtime. + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetTestingDescriptorPool(), {})); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + return CelTestContext::CreateFromRuntime( + std::move(runtime), + /*options=*/{.expression_source = + test::CelExpressionSource::FromCheckedExpr( + std::move(checked_expr))}); + }); +} // namespace cel::testing