Skip to content

Commit

Permalink
🐛 📝 DAT: verify that a destination is able to write any ISO8601-compl…
Browse files Browse the repository at this point in the history 
…iant date string (airbytehq#5814)

* added format_reference.txt, edited edge_case_messages.txt, edge_case_catalog.json

Co-authored-by: Yurii Bidiuk <yura.bidyuk@gmail.com>
  • Loading branch information
@bazarnov @yurii-bidiuk
bazarnov and yurii-bidiuk committed Dec 1, 2021
1 parent 8775cbe commit e4e95e3
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 5 deletions.
28 changes: 26 additions & 2 deletions ...main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,20 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.StreamSupport;
import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -1022,12 +1027,16 @@ protected void assertSameMessages(final List<AirbyteMessage> expected,
.map(AirbyteMessage::getRecord)
.peek(recordMessage -> recordMessage.setEmittedAt(null))
.map(recordMessage -> pruneAirbyteInternalFields ? safePrune(recordMessage) : recordMessage)
.map(recordMessage -> recordMessage.getData())
.map(AirbyteRecordMessage::getData)
.peek(this::sortDataFields)
.sorted(Comparator.comparing(JsonNode::toString))
.collect(Collectors.toList());

final List<JsonNode> actualProcessed = actual.stream()
.map(recordMessage -> pruneAirbyteInternalFields ? safePrune(recordMessage) : recordMessage)
.map(recordMessage -> recordMessage.getData())
.map(AirbyteRecordMessage::getData)
.peek(this::sortDataFields)
.sorted(Comparator.comparing(JsonNode::toString))
.collect(Collectors.toList());

assertSameData(expectedProcessed, actualProcessed);
Expand Down Expand Up @@ -1065,6 +1074,21 @@ private void assertSameData(final List<JsonNode> expected, final List<JsonNode>
}
}

/**
* Method that will sort all fields by name and rewrite JsonNode in sorted order
*
* @param data - data node from AirbyteMessage
*/
private void sortDataFields(JsonNode data) {
var sortedFields = StreamSupport.stream(Spliterators.spliteratorUnknownSize(data.fields(),
Spliterator.ORDERED), false)
.sorted(Entry.comparingByKey(Comparator.comparing(String::toLowerCase)))
.collect(Collectors.toList());
((ObjectNode) data).removeAll();
IntStream.range(0, sortedFields.size())
.forEach(i -> ((ObjectNode) data).set(sortedFields.get(i).getKey(), sortedFields.get(i).getValue()));
}

// Allows subclasses to implement custom comparison asserts
protected void assertSameValue(final JsonNode expectedValue, final JsonNode actualValue) {
assertEquals(expectedValue, actualValue);
Expand Down
15 changes: 15 additions & 0 deletions ...te-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,21 @@
}
}
}
},
{
"name": "stream_with_dates",
"json_schema": {
"properties": {
"date": {
"type": "string",
"format": "date"
},
"datetime": {
"type": "string",
"format": "date-time"
}
}
}
}
]
}
50 changes: 47 additions & 3 deletions ...te-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,56 @@
{"type": "RECORD", "record": {"stream": "reserved_keywords", "emitted_at": 1602637589000, "data": { "order" : "ascending" }}}
{"type": "RECORD", "record": {"stream": "groups", "emitted_at": 1602637589000, "data": { "authorization" : "into" }}}
{"type": "RECORD", "record": {"stream": "ProperCase", "emitted_at": 1602637589000, "data": { "ProperCase" : true }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589200, "data": { "some_id" : 101, "some_field" : "some_field_1", "some_next_field" : "some_next_field_1" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589200, "data": { "some_id" : 101, "some_next_field" : "some_next_field_1", "some_field" : "some_field_1" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589250, "data": { "some_id" : 102, "some_field" : "some_field_2" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589300, "data": { "some_id" : 103, "some_next_field" : "some_next_field_3" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589350, "data": { "some_id" : 104 }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589400, "data": { "some_id" : 201, "next_field_name" : "next_field_name_1" }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589450, "data": { "some_id" : 202, "next_field_name" : "next_field_name_2" }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589400, "data": { "next_field_name" : "next_field_name_1", "some_id" : 201 }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589450, "data": { "next_field_name" : "next_field_name_2", "some_id" : 202 }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589500, "data": { "some_id" : 203 }}}
{"type": "RECORD", "record": {"stream": "stream_with_binary_data", "emitted_at": 1602637589500, "data": { "some_id" : 303, "binary_field_name":"dGVzdA==" }}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "2021-1-1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "1/1/2021"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "20210101"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "2021-01-01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "01/01/2021"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T09"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T0930"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021/01/01 09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 09:30:26.123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 09:30:26.123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010,123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010.123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1 01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021 01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01-01:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01-01:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01Z"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01Z"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01+0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01+0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01+1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01+1"}}}
{"type": "STATE", "state": { "data": {"start_date": "2020-09-02"}}}
46 changes: 46 additions & 0 deletions airbyte-integrations/bases/standard-destination-test/src/main/resources/format_reference.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
VALID FORMATS:

"2021-1-1"
"1/1/2021"
"20210101"
"2021-01-01"
"01/01/2021"
"2021-01-01T09"
"20210101T0930"
"2021-01-01 09:30"
"2021/01/01 09:30"
"01/01/2021 09:30"
"20210101T093010"
"2021-01-01T09:30"
"2021-1-1 01:01:01"
"1/1/2021 01:01:01"
"2021-01-01 09:30:26.123"
"01/01/2021 09:30:26.123"
"20210101T093010,123"
"20210101T093010.123"
"2021-1-1 01:01:01 +1"
"1/1/2021 01:01:01 +1"
"2021-01-01T01:01:01-01:00"
"01/01/2021T01:01:01-01:00"
"2021-1-1T01:01:01 +1:00"
"1/1/2021T01:01:01 +1:00"
"2021-01-01 01:01:01"
"01/01/2021 01:01:01"
"2021-01-01 01:01:01 +0000"
"01/01/2021 01:01:01 +0000"
"2021-01-01T01:01:01Z"
"01/01/2021T01:01:01Z"
"2021-01-01T01:01:01 +1:00"
"01/01/2021T01:01:01 +1:00"
"2021-01-01 01:01:01 UTC"
"01/01/2021 01:01:01 UTC"
"2021-01-01T01:01:01 UTC"
"01/01/2021T01:01:01 UTC"
"2021-01-01T01:01:01 +1"
"01/01/2021T01:01:01 +1"
"2021-01-01T01:01:01 +0000"
"01/01/2021T01:01:01 +0000"
"2021-01-01T01:01:01+0000"
"01/01/2021T01:01:01+0000"
"2021-01-01T01:01:01+1"
"01/01/2021T01:01:01+1"

0 comments on commit e4e95e3

Please sign in to comment.