Skip to content

Commit

Permalink
NIFI-8023 Updated Record Reader tests to avoid additional conversion …
Browse files Browse the repository at this point in the history 
…prior to comparing expected results
  • Loading branch information
@exceptionfactory
exceptionfactory committed Jan 20, 2021
1 parent 98a9e69 commit e6b45b7
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 42 deletions.
12 changes: 7 additions & 5 deletions ...ization-services/src/test/java/org/apache/nifi/avro/TestAvroReaderWithEmbeddedSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.sql.Date;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
Expand Down Expand Up @@ -75,9 +77,10 @@ public void testNullableLogicalTypes() throws IOException, ParseException, Malfo
private void testLogicalTypes(Schema schema) throws ParseException, IOException, MalformedRecordException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();

final String expectedTime = "2017-04-04 14:20:33.000";
final String expectedDate = "2017-04-04";
final String expectedTime = String.format("%s 14:20:33.000", expectedDate);
final DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
df.setTimeZone(TimeZone.getTimeZone("gmt"));
df.setTimeZone(TimeZone.getTimeZone(ZoneOffset.UTC));
final long timeLong = df.parse(expectedTime).getTime();

final long secondsSinceMidnight = 33 + (20 * 60) + (14 * 60 * 60);
Expand Down Expand Up @@ -120,9 +123,8 @@ private void testLogicalTypes(Schema schema) throws ParseException, IOException,
assertEquals(new java.sql.Time(millisSinceMidnight), record.getValue("timeMicros"));
assertEquals(new java.sql.Timestamp(timeLong), record.getValue("timestampMillis"));
assertEquals(new java.sql.Timestamp(timeLong), record.getValue("timestampMicros"));
final DateFormat noTimeOfDayDateFormat = new SimpleDateFormat("yyyy-MM-dd");
noTimeOfDayDateFormat.setTimeZone(TimeZone.getTimeZone("gmt"));
assertEquals(noTimeOfDayDateFormat.format(new java.sql.Date(timeLong)), noTimeOfDayDateFormat.format(record.getValue("date")));

assertEquals(Date.valueOf(expectedDate), record.getValue("date"));
assertEquals(bigDecimal, record.getValue("decimal"));
}
}
Expand Down
43 changes: 20 additions & 23 deletions ...-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.TimeZone;
Expand Down Expand Up @@ -98,25 +97,24 @@ public void testUTF8() throws IOException, MalformedRecordException {

@Test
public void testDate() throws IOException, MalformedRecordException {
final String text = "date\n11/30/1983";
final String localDate = "1983-11-30";
final String localDateFormat = "yyyy-MM-dd";
final String dateField = "date";

final String text = String.format("%s\n%s", dateField, localDate);

final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);

for (final boolean coerceTypes : new boolean[] {true, false}) {
try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false,
"MM/dd/yyyy", RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {
try (final InputStream inputStream = new ByteArrayInputStream(text.getBytes());
final CSVRecordReader reader = new CSVRecordReader(inputStream, Mockito.mock(ComponentLog.class), schema, format, true, false,
localDateFormat, RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {

final Record record = reader.nextRecord(coerceTypes, false);
final java.sql.Date date = (Date) record.getValue("date");
final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("gmt"));
calendar.setTimeInMillis(date.getTime());

assertEquals(1983, calendar.get(Calendar.YEAR));
assertEquals(10, calendar.get(Calendar.MONTH));
assertEquals(30, calendar.get(Calendar.DAY_OF_MONTH));
final Object date = record.getValue(dateField);
assertEquals(Date.valueOf(localDate), date);
}
}
}
Expand All @@ -143,24 +141,23 @@ public void testBigDecimal() throws IOException, MalformedRecordException {

@Test
public void testDateNoCoersionExpectedFormat() throws IOException, MalformedRecordException {
final String text = "date\n11/30/1983";
final String localDate = "1983-11-30";
final String localDateFormat = "yyyy-MM-dd";
final String dateField = "date";

final String text = String.format("%s\n%s", dateField, localDate);

final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);

try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false,
"MM/dd/yyyy", RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {
try (final InputStream inputStream = new ByteArrayInputStream(text.getBytes());
final CSVRecordReader reader = new CSVRecordReader(inputStream, Mockito.mock(ComponentLog.class), schema, format, true, false,
localDateFormat, RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {

final Record record = reader.nextRecord(false, false);
final java.sql.Date date = (Date) record.getValue("date");
final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("gmt"));
calendar.setTimeInMillis(date.getTime());

assertEquals(1983, calendar.get(Calendar.YEAR));
assertEquals(10, calendar.get(Calendar.MONTH));
assertEquals(30, calendar.get(Calendar.DAY_OF_MONTH));
final Object date = record.getValue(dateField);
assertEquals(Date.valueOf(localDate), date);
}
}

Expand Down
25 changes: 11 additions & 14 deletions ...-serialization-services/src/test/java/org/apache/nifi/csv/TestJacksonCSVRecordReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@
import java.io.InputStream;
import java.sql.Date;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.TimeZone;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
Expand Down Expand Up @@ -85,24 +83,23 @@ public void testUTF8() throws IOException, MalformedRecordException {

@Test
public void testDate() throws IOException, MalformedRecordException {
final String text = "date\n11/30/1983";
final String localDate = "1983-11-30";
final String localDateFormat = "yyyy-MM-dd";
final String dateField = "date";

final String text = String.format("%s\n%s", dateField, localDate);

final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
fields.add(new RecordField(dateField, RecordFieldType.DATE.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);

try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
final JacksonCSVRecordReader reader = new JacksonCSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false,
"MM/dd/yyyy", RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {
try (final InputStream inputStream = new ByteArrayInputStream(text.getBytes());
final JacksonCSVRecordReader reader = new JacksonCSVRecordReader(inputStream, Mockito.mock(ComponentLog.class), schema, format, true, false,
localDateFormat, RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {

final Record record = reader.nextRecord();
final Date date = (Date) record.getValue("date");
final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("gmt"));
calendar.setTimeInMillis(date.getTime());

assertEquals(1983, calendar.get(Calendar.YEAR));
assertEquals(10, calendar.get(Calendar.MONTH));
assertEquals(30, calendar.get(Calendar.DAY_OF_MONTH));
final Object date = record.getValue(dateField);
assertEquals(Date.valueOf(localDate), date);
}
}

Expand Down

0 comments on commit e6b45b7

Please sign in to comment.