diff --git a/ingestion-core/src/main/java/com/mozilla/telemetry/ingestion/core/transform/PubsubMessageToObjectNode.java b/ingestion-core/src/main/java/com/mozilla/telemetry/ingestion/core/transform/PubsubMessageToObjectNode.java index b32c51a8e..7da6dd7ee 100644 --- a/ingestion-core/src/main/java/com/mozilla/telemetry/ingestion/core/transform/PubsubMessageToObjectNode.java +++ b/ingestion-core/src/main/java/com/mozilla/telemetry/ingestion/core/transform/PubsubMessageToObjectNode.java @@ -330,8 +330,12 @@ private void processField(String jsonFieldName, Field field, JsonNode value, Obj ObjectNode additionalProperties) { final String bqFieldName = field.getName(); - // A record of key and value indicates we need to transformForBqSchema a map to an array. - if (isMapType(field)) { + // null is valid for any type except an element of a list + if (value.isNull()) { + updateParent(parent, jsonFieldName, bqFieldName, value); + + // A record of key and value indicates we need to transformForBqSchema a map to an array. + } else if (isMapType(field)) { expandMapType(jsonFieldName, (ObjectNode) value, field, parent, additionalProperties); // A record with a single "list" field and a list value should be expanded appropriately. @@ -516,10 +520,7 @@ private void expandNestedListType(String jsonFieldName, ArrayNode value, Field f * field should be put to {@code additional_properties}. */ private Optional coerceToBqType(JsonNode o, Field field) { - if (o.isNull()) { - // null is valid for any type, just not as an element of a list - return Optional.of(o); - } else if (field.getMode() == Field.Mode.REPEATED) { + if (field.getMode() == Field.Mode.REPEATED) { if (o.isArray()) { // We have not yet observed a case where an array type contains values that cannot be // coerced to appropriate values, but if it does this will throw NoSuchElementException diff --git a/ingestion-core/src/test/resources/testdata/payload-format-input.ndjson b/ingestion-core/src/test/resources/testdata/payload-format-input.ndjson index 2f4757c73..2bb3bf12b 100644 --- a/ingestion-core/src/test/resources/testdata/payload-format-input.ndjson +++ b/ingestion-core/src/test/resources/testdata/payload-format-input.ndjson @@ -22,6 +22,6 @@ {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_tuple":[{"key":"value"},"x",6]}} {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_nested_list":[["a","b","c"]],"test_nested_csv":[{"list":"a,b,c"}],"test_list":["a","b","c"]}} {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_list":null,"test_nested_list":[null],"test_record":null}} -{"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_record":{"key":null}}} +{"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_record":{"key":null},"test_map":null}} {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_int64 ":null,"test_int64":7,"test_int64_":"invalid","test_int64+":8}} {"attributeMap":{"document_namespace":"live-sink","document_type":"test","document_version":"1"},"payload":{"test_map":{"key":null,"value":null,"other":"other"}}}