Skip to content

Commit

Permalink
Added Serialization/Deserialization test, added blank row test file, …
Browse files Browse the repository at this point in the history 
…cleaned up compressed file test
  • Loading branch information
Megan Foss committed Oct 15, 2021
1 parent 2262cb1 commit 2d17f1b
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 2 deletions.
67 changes: 65 additions & 2 deletions ...idth/src/test/java/org/apache/drill/exec/store/fixedwidth/TestFixedwidthRecordReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import java.time.LocalTime;

import static org.apache.drill.test.QueryTestUtil.generateCompressedFile;
import static org.junit.Assert.assertEquals;

@Category(RowSetTests.class)
public class TestFixedwidthRecordReader extends ClusterTest {
Expand Down Expand Up @@ -98,20 +99,82 @@ public void testExplicitQuery() throws Exception {
}

//Test Serialization/Deserialization
@Test
public void testSerDe() throws Exception {
String sql = "SELECT COUNT(*) FROM dfs.`fwf/test.fwf`";
String plan = queryBuilder().sql(sql).explainJson();
long cnt = queryBuilder().physical(plan).singletonLong();
assertEquals(5L, cnt);
}

//Test Compressed File
@Test
public void testStarQueryWithCompressedFile() throws Exception {
generateCompressedFile("fwf/test.fwf", "zip", "fwf/test.fwf.zip" );

String sql = "SELECT * FROM dfs.`fwf/test.fwf.zip`";
System.out.println("Compressed file generated");
RowSet results = client.queryBuilder().sql(sql).rowSet();
RowSet expected = setupTestData();
new RowSetComparison(expected).verifyAndClearAll(results);
}

// Test Entering invalid schemata - incorrect limits
// Undefined field, what happens
// Parse invalid file, make sure correct error


@Test
public void testOutOfOrder() throws Exception{
String sql = "SELECT Address, DateTime, `Date`, Letter FROM cp.`fwf/test.fwf`";
QueryBuilder q = client.queryBuilder().sql(sql);
RowSet results = q.rowSet();

TupleMetadata expectedSchema = new SchemaBuilder()
.addNullable("Address", TypeProtos.MinorType.INT)
.addNullable("DateTime", TypeProtos.MinorType.TIMESTAMP)
.addNullable("Date", TypeProtos.MinorType.DATE)
.addNullable("Letter", TypeProtos.MinorType.VARCHAR)
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow(567, Instant.parse("2021-02-10T15:30:27.00Z"), LocalDate.parse("2021-02-10"), "test")
.addRow(890, Instant.parse("2021-07-27T16:40:15.00Z"), LocalDate.parse("2021-07-27"), "TEST")
.addRow(111, Instant.parse("1111-11-11T16:28:43.11Z"), LocalDate.parse("1111-11-11"), "abcd")
.addRow(222, Instant.parse("2222-01-23T03:22:22.22Z"), LocalDate.parse("2222-01-22"), "efgh")
.addRow(333, Instant.parse("3333-02-01T06:33:33.33Z"), LocalDate.parse("3333-02-01"), "ijkl")
.addRow(444, Instant.parse("4444-03-02T07:44:44.44Z"), LocalDate.parse("4444-03-02"), "mnop")
.addRow(555, Instant.parse("5555-04-03T07:55:55.55Z"), LocalDate.parse("5555-04-03"), "qrst")
.addRow(666, Instant.parse("6666-05-04T08:01:01.01Z"), LocalDate.parse("6666-05-04"), "uvwx")
.addRow(777, Instant.parse("7777-06-05T09:11:11.11Z"), LocalDate.parse("7777-06-05"), "yzzz")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.addRow(888, Instant.parse("8888-07-07T10:22:22.22Z"), LocalDate.parse("8888-07-06"), "aabb")
.build();

new RowSetComparison(expected).verifyAndClearAll(results);
}

// How should we be handling an empty/blank row?
@Test
public void testEmptyRow() throws Exception {
String sql = "SELECT * FROM cp.`fwf/test_blankrow.fwf`";
RowSet results = client.queryBuilder().sql(sql).rowSet();
RowSet expected = setupTestData();
new RowSetComparison(expected).verifyAndClearAll(results);
}

//

private RowSet setupTestData(){
TupleMetadata expectedSchema = new SchemaBuilder()
Expand Down
26 changes: 26 additions & 0 deletions contrib/format-fixedwidth/src/test/resources/fwf/test_blankrow.fwf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
12.34 test 567 02-10-2021 10:30:27 02-10-2021T10:30:27.00Z
56.78 TEST 890 07-27-2021 12:40:15 07-27-2021T12:40:15.00Z
11.11 abcd 111 11-11-1111 11:11:11 11-11-1111T11:11:11.11Z
22.22 efgh 222 01-22-2222 22:22:22 01-22-2222T22:22:22.22Z
33.33 ijkl 333 02-01-3333 01:33:33 02-01-3333T01:33:33.33Z
44.44 mnop 444 03-02-4444 02:44:44 03-02-4444T02:44:44.44Z
55.55 qrst 555 04-03-5555 03:55:55 04-03-5555T03:55:55.55Z
66.66 uvwx 666 05-04-6666 04:01:01 05-04-6666T04:01:01.01Z
77.77 yzzz 777 06-05-7777 05:11:11 06-05-7777T05:11:11.11Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z

88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z
88.88 aabb 888 07-06-8888 06:22:22 07-07-8888T06:22:22.22Z

0 comments on commit 2d17f1b

Please sign in to comment.