Give a specific category to each test. (#4965)

* Give a unique category to each test.

This change introduce a TestCategory enum to ConformanceRequest. Existing tests
are divided into three categories: binary format test, json format test and json
format (ignore unknown when parsing) test. For the previous two categories, there
is no change to existing testee programs. For tests with the last category, testee programs
should either enable ignoring unknown field during json parsing or skip the test.

* Fix python test

* Fix java

* Fix csharp

* Update document

* Update csharp generated code

conformance/ConformanceJava.java

@@ -235,8 +235,12 @@ private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest re
         try {
           TestMessagesProto3.TestAllTypesProto3.Builder builder = 
               TestMessagesProto3.TestAllTypesProto3.newBuilder();
-          JsonFormat.parser().usingTypeRegistry(typeRegistry)
-              .merge(request.getJsonPayload(), builder);
+          JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(typeRegistry);
+          if (request.getTestCategory()
+              == Conformance.TestCategory.JSON_IGNORE_UNKNOWN_PARSING_TEST) {
+            parser = parser.ignoringUnknownFields();
+          }
+          parser.merge(request.getJsonPayload(), builder);
           testMessage = builder.build();
         } catch (InvalidProtocolBufferException e) {
           return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build();

conformance/conformance.proto

@@ -57,6 +57,17 @@ enum WireFormat {
   JSON = 2;
 }
 
+enum TestCategory {
+  BINARY_TEST = 0;  // Test binary wire format.
+  JSON_TEST = 1;  // Test json wire format.
+  // Similar to JSON_TEST. However, during parsing json, testee should ignore
+  // unknown fields. This feature is optional. Each implementation can descide
+  // whether to support it.  See
+  // https://developers.google.com/protocol-buffers/docs/proto3#json_options
+  // for more detail.
+  JSON_IGNORE_UNKNOWN_PARSING_TEST = 2;  
+}
+
 // Represents a single test case's input.  The testee should:
 //
 //   1. parse this proto (which should always succeed)
@@ -83,7 +94,10 @@ message ConformanceRequest {
   // protobuf_test_messages.proto2.TestAllTypesProto2.
   string message_type = 4;
 
-  bool ignore_unknown_json = 5;
+  // Each test is given a specific test category. Some category may need spedific
+  // support in testee programs. Refer to the defintion of TestCategory for
+  // more information.
+  TestCategory test_category = 5;
 }
 
 // Represents a single test case's output.

conformance/conformance_cpp.cc

@@ -46,6 +46,7 @@ using google::protobuf::DescriptorPool;
 using google::protobuf::Message;
 using google::protobuf::MessageFactory;
 using google::protobuf::util::BinaryToJsonString;
+using google::protobuf::util::JsonParseOptions;
 using google::protobuf::util::JsonToBinaryString;
 using google::protobuf::util::NewTypeResolverForDescriptorPool;
 using google::protobuf::util::Status;
@@ -112,8 +113,13 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
 
     case ConformanceRequest::kJsonPayload: {
       string proto_binary;
+      JsonParseOptions options;
+      options.ignore_unknown_fields =
+          (request.test_category() ==
+              conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST);
       Status status = JsonToBinaryString(type_resolver, *type_url,
-                                         request.json_payload(), &proto_binary);
+                                         request.json_payload(), &proto_binary,
+                                         options);
       if (!status.ok()) {
         response->set_parse_error(string("Parse error: ") +
                                   status.error_message().as_string());

conformance/conformance_php.php

@@ -3,6 +3,7 @@
 require_once("Conformance/WireFormat.php");
 require_once("Conformance/ConformanceResponse.php");
 require_once("Conformance/ConformanceRequest.php");
+require_once("Conformance/TestCategory.php");
 require_once("Protobuf_test_messages/Proto3/ForeignMessage.php");
 require_once("Protobuf_test_messages/Proto3/ForeignEnum.php");
 require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3.php");
@@ -12,6 +13,7 @@
 require_once("GPBMetadata/Conformance.php");
 require_once("GPBMetadata/Google/Protobuf/TestMessagesProto3.php");
 
+use  \Conformance\TestCategory;
 use  \Conformance\WireFormat;
 
 if (!ini_get("date.timezone")) {
@@ -39,7 +41,9 @@ function doTest($request)
         trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR);
       }
     } elseif ($request->getPayload() == "json_payload") {
-      $ignore_json_unknown = $request->getIgnoreUnknownJson();
+      $ignore_json_unknown =
+          ($request->getTestCategory() ==
+              TestCategory::JSON_IGNORE_UNKNOWN_PARSING_TEST);
       try {
           $test_message->mergeFromJsonString($request->getJsonPayload(),
                                              $ignore_json_unknown);

conformance/conformance_python.py

@@ -78,7 +78,11 @@ def do_test(request):
 
     elif request.WhichOneof('payload') == 'json_payload':
       try:
-        json_format.Parse(request.json_payload, test_message)
+        ignore_unknown_fields = \
+            request.test_category == \
+                conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST
+        json_format.Parse(request.json_payload, test_message,
+                          ignore_unknown_fields)
       except Exception as e:
         response.parse_error = str(e)
         return response

conformance/conformance_test.cc

@@ -193,10 +193,11 @@ namespace protobuf {
 
 ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
     ConformanceLevel level, conformance::WireFormat input_format,
-    conformance::WireFormat output_format, bool is_proto3,
+    conformance::WireFormat output_format,
+    conformance::TestCategory test_category,
+    bool is_proto3,
     const string& test_name, const string& input)
-    : level_(level), input_format_(input_format),
-      output_format_(output_format), is_proto3_(is_proto3) {
+    : level_(level), is_proto3_(is_proto3) {
   auto newTestMessage = [&is_proto3]() {
     Message* newMessage;
     if (is_proto3) {
@@ -243,6 +244,8 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
       GOOGLE_LOG(FATAL) << "Unspecified output format";
   }
 
+  request_.set_test_category(test_category);
+
   test_name_ = ConformanceLevelToString(level) + rname +
                input_format_string + test_name +
                output_format_string;
@@ -465,6 +468,7 @@ void ConformanceTestSuite::ExpectParseFailureForProtoWithProtoVersion (
   ConformanceRequest request;
   ConformanceResponse response;
   request.set_protobuf_payload(proto);
+  request.set_test_category(conformance::BINARY_TEST);
   if (isProto3) {
     request.set_message_type("protobuf_test_messages.proto3.TestAllTypesProto3");
   } else {
@@ -511,11 +515,13 @@ void ConformanceTestSuite::RunValidJsonTest(
     const string& equivalent_text_format) {
   ConformanceRequestSetting setting1(
       level, conformance::JSON, conformance::PROTOBUF,
+      conformance::JSON_TEST,
       true, test_name, input_json);
   RunValidInputTest(setting1, equivalent_text_format);
 
   ConformanceRequestSetting setting2(
       level, conformance::JSON, conformance::JSON,
+      conformance::JSON_TEST,
       true, test_name, input_json);
   RunValidInputTest(setting2, equivalent_text_format);
 }
@@ -525,6 +531,7 @@ void ConformanceTestSuite::RunValidJsonTestWithProtobufInput(
     const string& equivalent_text_format) {
   ConformanceRequestSetting setting(
       level, conformance::PROTOBUF, conformance::JSON,
+      conformance::JSON_TEST,
       true, test_name, input.SerializeAsString());
   RunValidInputTest(setting, equivalent_text_format);
 }
@@ -534,8 +541,8 @@ void ConformanceTestSuite::RunValidJsonIgnoreUnknownTest(
     const string& equivalent_text_format) {
   ConformanceRequestSetting setting(
       level, conformance::JSON, conformance::PROTOBUF,
+      conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST,
       true, test_name, input_json);
-  setting.SetIgnoreUnknownJson(true);
   RunValidInputTest(setting, equivalent_text_format);
 }
 
@@ -545,12 +552,14 @@ void ConformanceTestSuite::RunValidProtobufTest(
     bool isProto3) {
   ConformanceRequestSetting setting1(
       level, conformance::PROTOBUF, conformance::PROTOBUF,
+      conformance::BINARY_TEST,
       isProto3, test_name, input_protobuf);
   RunValidInputTest(setting1, equivalent_text_format);
 
   if (isProto3) {
     ConformanceRequestSetting setting2(
         level, conformance::PROTOBUF, conformance::JSON,
+        conformance::BINARY_TEST,
         true, test_name, input_protobuf);
     RunValidInputTest(setting2, equivalent_text_format);
   }
@@ -561,6 +570,7 @@ void ConformanceTestSuite::RunValidBinaryProtobufTest(
     const string& input_protobuf, bool isProto3) {
   ConformanceRequestSetting setting(
       level, conformance::PROTOBUF, conformance::PROTOBUF,
+      conformance::BINARY_TEST,
       isProto3, test_name, input_protobuf);
   RunValidBinaryInputTest(setting, input_protobuf);
 }

conformance/conformance_test.h

@@ -152,7 +152,9 @@ class ConformanceTestSuite {
    public:
     ConformanceRequestSetting(
         ConformanceLevel level, conformance::WireFormat input_format,
-        conformance::WireFormat output_format, bool is_proto3,
+        conformance::WireFormat output_format,
+        conformance::TestCategory test_category,
+        bool is_proto3,
         const string& test_name, const string& input);
 
    Message* GetTestMessage() const;
@@ -169,14 +171,8 @@ class ConformanceTestSuite {
      return level_;
    }
 
-   void SetIgnoreUnknownJson(bool ignore_unknown_json) {
-     request_.set_ignore_unknown_json(ignore_unknown_json);
-   }
-
    private:
     ConformanceLevel level_;
-    conformance::WireFormat input_format_;
-    conformance::WireFormat output_format_;
     bool is_proto3_;
     string test_name_;
     conformance::ConformanceRequest request_;

conformance/failure_list_cpp.txt

@@ -54,9 +54,3 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput

conformance/failure_list_csharp.txt

@@ -1,8 +1,2 @@
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput

conformance/failure_list_java.txt

@@ -45,9 +45,3 @@ Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValu
 Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
 Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
 Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput

conformance/failure_list_python-post26.txt

@@ -1,8 +1,2 @@
 JsonInput.StringFieldSurrogateInWrongOrder
 JsonInput.StringFieldUnpairedHighSurrogate
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput

conformance/failure_list_python.txt

@@ -19,9 +19,3 @@ Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_0
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_1
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_2
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_3
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput

conformance/failure_list_python_cpp.txt

@@ -52,9 +52,3 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput