Allow complex types to be null when transforming to payload format (#…

…1318)
mozilla · Jul 6, 2020 · 593f0db · 593f0db
1 parent 59e1dd9
commit 593f0db
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/...c/main/java/com/mozilla/telemetry/ingestion/core/transform/PubsubMessageToObjectNode.java b/...c/main/java/com/mozilla/telemetry/ingestion/core/transform/PubsubMessageToObjectNode.java
@@ -330,8 +330,12 @@ private void processField(String jsonFieldName, Field field, JsonNode value, Obj
         ObjectNode additionalProperties) {
       final String bqFieldName = field.getName();
 
-      // A record of key and value indicates we need to transformForBqSchema a map to an array.
-      if (isMapType(field)) {
+      // null is valid for any type except an element of a list
+      if (value.isNull()) {
+        updateParent(parent, jsonFieldName, bqFieldName, value);
+
+        // A record of key and value indicates we need to transformForBqSchema a map to an array.
+      } else if (isMapType(field)) {
         expandMapType(jsonFieldName, (ObjectNode) value, field, parent, additionalProperties);
 
         // A record with a single "list" field and a list value should be expanded appropriately.
@@ -516,10 +520,7 @@ private void expandNestedListType(String jsonFieldName, ArrayNode value, Field f
      * field should be put to {@code additional_properties}.
      */
     private Optional<JsonNode> coerceToBqType(JsonNode o, Field field) {
-      if (o.isNull()) {
-        // null is valid for any type, just not as an element of a list
-        return Optional.of(o);
-      } else if (field.getMode() == Field.Mode.REPEATED) {
+      if (field.getMode() == Field.Mode.REPEATED) {
         if (o.isArray()) {
           // We have not yet observed a case where an array type contains values that cannot be
           // coerced to appropriate values, but if it does this will throw NoSuchElementException

diff --git a/ingestion-core/src/test/resources/testdata/payload-format-input.ndjson b/ingestion-core/src/test/resources/testdata/payload-format-input.ndjson
@@ -22,6 +22,6 @@
 {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_tuple":[{"key":"value"},"x",6]}}
 {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_nested_list":[["a","b","c"]],"test_nested_csv":[{"list":"a,b,c"}],"test_list":["a","b","c"]}}
 {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_list":null,"test_nested_list":[null],"test_record":null}}
-{"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_record":{"key":null}}}
+{"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_record":{"key":null},"test_map":null}}
 {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_int64 ":null,"test_int64":7,"test_int64_":"invalid","test_int64+":8}}
 {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_map":{"key":null,"value":null,"other":"other"}}}