Support for error message assertion.

testing/testrunner/BUILD

@@ -42,6 +42,7 @@ cc_library(
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings:string_view",
+        "@com_google_cel_spec//proto/cel/expr:value_cc_proto",
         "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto",
         "@com_google_protobuf//:differencer",
         "@com_google_protobuf//:protobuf",

testing/testrunner/runner_lib.cc

@@ -16,6 +16,7 @@
 #include <memory>
 #include <utility>
 
+#include "cel/expr/eval.pb.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/string_view.h"
@@ -226,13 +227,29 @@ void TestRunner::AssertValue(const cel::Value& computed,
   EXPECT_THAT(expected_value_proto, MatchesValue(computed_expr_value));
 }
 
+void TestRunner::AssertError(const cel::Value& computed,
+                             const TestOutput& output) {
+  if (!computed.IsError()) {
+    ADD_FAILURE() << "Expected error but got value: " << computed.DebugString();
+    return;
+  }
+  absl::Status computed_status = computed.AsError()->ToStatus();
+  // We selected the first error in the set for comparison because there is only
+  // one runtime error that is reported even if there are multiple errors in the
+  // critical path.
+  ASSERT_TRUE(output.eval_error().errors_size() == 1)
+      << "Expected exactly one error but got: "
+      << output.eval_error().errors_size();
+  ASSERT_EQ(computed_status.message(), output.eval_error().errors(0).message());
+}
+
 void TestRunner::Assert(const cel::Value& computed, const TestCase& test_case,
                         google::protobuf::Arena* arena) {
   TestOutput output = test_case.output();
   if (output.has_result_value() || output.has_result_expr()) {
     AssertValue(computed, output, arena);
   } else if (output.has_eval_error()) {
-    ADD_FAILURE() << "Error assertion not implemented yet.";
+    AssertError(computed, output);
   } else if (output.has_unknown()) {
     ADD_FAILURE() << "Unknown assertions not implemented yet.";
   } else {

testing/testrunner/runner_lib.h

@@ -53,6 +53,9 @@ class TestRunner {
                    const cel::expr::conformance::test::TestOutput& output,
                    google::protobuf::Arena* arena);
 
+  void AssertError(const cel::Value& computed,
+                   const cel::expr::conformance::test::TestOutput& output);
+
   std::unique_ptr<cel::test::CelTestContext> test_context_;
 };
 

testing/testrunner/runner_lib_test.cc

@@ -450,5 +450,54 @@ TEST(TestRunnerCustomCompilerTest,
       std::move(runtime), /*options=*/{.checked_expr = checked_expr}));
   EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case));
 }
+
+TEST_F(TestRunnerTest, BasicTestWithErrorAssertion) {
+  // Compile the expression.
+  ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result,
+                       compiler_->Compile("x + y"));
+  CheckedExpr checked_expr;
+  ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr),
+              absl_testing::IsOk());
+  // Create a runtime.
+  ASSERT_OK_AND_ASSIGN(std::unique_ptr<const cel::Runtime> runtime,
+                       CreateTestRuntime());
+  TestCase test_case = ParseTextProtoOrDie<TestCase>(R"pb(
+    input {
+      key: "x"
+      value { value { int64_value: 1 } }
+    }
+    output {
+      eval_error {
+        errors { message: "No value with name \"y\" found in Activation" }
+      }
+    }
+  )pb");
+  TestRunner test_runner(CelTestContext::CreateFromRuntime(
+      std::move(runtime), /*options=*/{.checked_expr = checked_expr}));
+  EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case));
+}
+
+TEST_F(TestRunnerTest, BasicTestFailsWhenExpectingErrorButGotValue) {
+  // Compile the expression.
+  ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result,
+                       compiler_->Compile("1 + 1"));
+  CheckedExpr checked_expr;
+  ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr),
+              absl_testing::IsOk());
+  // Create a runtime.
+  ASSERT_OK_AND_ASSIGN(std::unique_ptr<const cel::Runtime> runtime,
+                       CreateTestRuntime());
+  TestCase test_case = ParseTextProtoOrDie<TestCase>(R"pb(
+    output {
+      eval_error {
+        errors { message: "No value with name \"y\" found in Activation" }
+      }
+    }
+  )pb");
+  TestRunner test_runner(CelTestContext::CreateFromRuntime(
+      std::move(runtime), /*options=*/{.checked_expr = checked_expr}));
+  EXPECT_NONFATAL_FAILURE(test_runner.RunTest(test_case),
+                          "Expected error but got value");
+}
 }  // namespace
 }  // namespace cel::test