diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD index 39087b598..523f275de 100644 --- a/testing/testrunner/BUILD +++ b/testing/testrunner/BUILD @@ -42,6 +42,7 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr:value_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", "@com_google_protobuf//:differencer", "@com_google_protobuf//:protobuf", diff --git a/testing/testrunner/runner_lib.cc b/testing/testrunner/runner_lib.cc index e0fe723dd..26aa09ddf 100644 --- a/testing/testrunner/runner_lib.cc +++ b/testing/testrunner/runner_lib.cc @@ -16,6 +16,7 @@ #include #include +#include "cel/expr/eval.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -226,13 +227,29 @@ void TestRunner::AssertValue(const cel::Value& computed, EXPECT_THAT(expected_value_proto, MatchesValue(computed_expr_value)); } +void TestRunner::AssertError(const cel::Value& computed, + const TestOutput& output) { + if (!computed.IsError()) { + ADD_FAILURE() << "Expected error but got value: " << computed.DebugString(); + return; + } + absl::Status computed_status = computed.AsError()->ToStatus(); + // We selected the first error in the set for comparison because there is only + // one runtime error that is reported even if there are multiple errors in the + // critical path. + ASSERT_TRUE(output.eval_error().errors_size() == 1) + << "Expected exactly one error but got: " + << output.eval_error().errors_size(); + ASSERT_EQ(computed_status.message(), output.eval_error().errors(0).message()); +} + void TestRunner::Assert(const cel::Value& computed, const TestCase& test_case, google::protobuf::Arena* arena) { TestOutput output = test_case.output(); if (output.has_result_value() || output.has_result_expr()) { AssertValue(computed, output, arena); } else if (output.has_eval_error()) { - ADD_FAILURE() << "Error assertion not implemented yet."; + AssertError(computed, output); } else if (output.has_unknown()) { ADD_FAILURE() << "Unknown assertions not implemented yet."; } else { diff --git a/testing/testrunner/runner_lib.h b/testing/testrunner/runner_lib.h index 895a07950..be8736da4 100644 --- a/testing/testrunner/runner_lib.h +++ b/testing/testrunner/runner_lib.h @@ -53,6 +53,9 @@ class TestRunner { const cel::expr::conformance::test::TestOutput& output, google::protobuf::Arena* arena); + void AssertError(const cel::Value& computed, + const cel::expr::conformance::test::TestOutput& output); + std::unique_ptr test_context_; }; diff --git a/testing/testrunner/runner_lib_test.cc b/testing/testrunner/runner_lib_test.cc index 2499a0b5c..f67c44242 100644 --- a/testing/testrunner/runner_lib_test.cc +++ b/testing/testrunner/runner_lib_test.cc @@ -450,5 +450,54 @@ TEST(TestRunnerCustomCompilerTest, std::move(runtime), /*options=*/{.checked_expr = checked_expr})); EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } + +TEST_F(TestRunnerTest, BasicTestWithErrorAssertion) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a runtime. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 1 } } + } + output { + eval_error { + errors { message: "No value with name \"y\" found in Activation" } + } + } + )pb"); + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} + +TEST_F(TestRunnerTest, BasicTestFailsWhenExpectingErrorButGotValue) { + // Compile the expression. + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler_->Compile("1 + 1")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + // Create a runtime. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + TestCase test_case = ParseTextProtoOrDie(R"pb( + output { + eval_error { + errors { message: "No value with name \"y\" found in Activation" } + } + } + )pb"); + TestRunner test_runner(CelTestContext::CreateFromRuntime( + std::move(runtime), /*options=*/{.checked_expr = checked_expr})); + EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case), + "Expected error but got value"); +} } // namespace } // namespace cel::test