Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
DRILL-7006: Add type conversion to row writers
Modifies the column metadata and writer abstractions to allow a type conversion "shim" to be specified as part of the schema, then inserted as part of the row set writer. Allows, say, setting an Int or Date from a string, parsing the string to obtain the proper data type to store in the vector. Type conversion not yet supported in the result set loader: some additional complexity needs to be resolved. Adds unit tests for this functionality. Refactors some existing tests to remove rough edges. closes #1623
- Loading branch information
1 parent
85c656e
commit 8fb85cd
Showing
13 changed files
with
654 additions
and
80 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
145 changes: 145 additions & 0 deletions
145
exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConvertor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,145 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.drill.test.rowSet.test; | ||
|
|
||
| import static org.apache.drill.test.rowSet.RowSetUtilities.strArray; | ||
| import static org.apache.drill.test.rowSet.RowSetUtilities.intArray; | ||
|
|
||
| import org.apache.drill.common.types.TypeProtos.MinorType; | ||
| import org.apache.drill.exec.record.metadata.ColumnMetadata; | ||
| import org.apache.drill.exec.record.metadata.SchemaBuilder; | ||
| import org.apache.drill.exec.record.metadata.TupleMetadata; | ||
| import org.apache.drill.exec.vector.accessor.ColumnConversionFactory; | ||
| import org.apache.drill.exec.vector.accessor.ScalarWriter; | ||
| import org.apache.drill.exec.vector.accessor.writer.AbstractWriteConvertor; | ||
| import org.apache.drill.exec.vector.accessor.writer.ConcreteWriter; | ||
| import org.apache.drill.test.SubOperatorTest; | ||
| import org.apache.drill.test.rowSet.RowSet; | ||
| import org.apache.drill.test.rowSet.RowSetBuilder; | ||
| import org.apache.drill.test.rowSet.RowSetUtilities; | ||
| import org.apache.drill.test.rowSet.RowSet.SingleRowSet; | ||
| import org.junit.Test; | ||
|
|
||
| /** | ||
| * Tests the column type convertor feature of the column metadata | ||
| * and of the RowSetWriter. | ||
| */ | ||
|
|
||
| public class TestColumnConvertor extends SubOperatorTest { | ||
|
|
||
| /** | ||
| * Simple type converter that allows string-to-int conversions. | ||
| * Inherits usual int value support from the base writer. | ||
| */ | ||
| public static class TestConvertor extends AbstractWriteConvertor { | ||
|
|
||
| public TestConvertor(ScalarWriter baseWriter) { | ||
| super(baseWriter); | ||
| } | ||
|
|
||
| @Override | ||
| public void setString(String value) { | ||
| setInt(Integer.parseInt(value)); | ||
| } | ||
|
|
||
| public static ColumnConversionFactory factory() { | ||
| return new ColumnConversionFactory() { | ||
| @Override | ||
| public ConcreteWriter newWriter(ColumnMetadata colDefn, | ||
| ConcreteWriter baseWriter) { | ||
| return new TestConvertor(baseWriter); | ||
| } | ||
| }; | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void testScalarConvertor() { | ||
|
|
||
| // Create the schema | ||
|
|
||
| TupleMetadata schema = new SchemaBuilder() | ||
| .add("n1", MinorType.INT) | ||
| .addNullable("n2", MinorType.INT) | ||
| .buildSchema(); | ||
|
|
||
| // Add a type convertor. Passed in as a factory | ||
| // since we must create a new one for each row set writer. | ||
|
|
||
| schema.metadata("n1").setTypeConverter(TestConvertor.factory()); | ||
| schema.metadata("n2").setTypeConverter(TestConvertor.factory()); | ||
|
|
||
| // Write data as both a string as an integer | ||
|
|
||
| RowSet actual = new RowSetBuilder(fixture.allocator(), schema) | ||
| .addRow("123", "12") | ||
| .addRow(234, 23) | ||
| .build(); | ||
|
|
||
| // Build the expected vector without a type convertor. | ||
|
|
||
| TupleMetadata expectedSchema = new SchemaBuilder() | ||
| .add("n1", MinorType.INT) | ||
| .addNullable("n2", MinorType.INT) | ||
| .buildSchema(); | ||
| final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema) | ||
| .addRow(123, 12) | ||
| .addRow(234, 23) | ||
| .build(); | ||
|
|
||
| // Compare | ||
|
|
||
| RowSetUtilities.verify(expected, actual); | ||
| } | ||
|
|
||
| @Test | ||
| public void testArrayConvertor() { | ||
|
|
||
| // Create the schema | ||
|
|
||
| TupleMetadata schema = new SchemaBuilder() | ||
| .addArray("n", MinorType.INT) | ||
| .buildSchema(); | ||
|
|
||
| // Add a type convertor. Passed in as a factory | ||
| // since we must create a new one for each row set writer. | ||
|
|
||
| schema.metadata("n").setTypeConverter(TestConvertor.factory()); | ||
|
|
||
| // Write data as both a string as an integer | ||
|
|
||
| RowSet actual = new RowSetBuilder(fixture.allocator(), schema) | ||
| .addSingleCol(strArray("123", "124")) | ||
| .addSingleCol(intArray(234, 235)) | ||
| .build(); | ||
|
|
||
| // Build the expected vector without a type convertor. | ||
|
|
||
| TupleMetadata expectedSchema = new SchemaBuilder() | ||
| .addArray("n", MinorType.INT) | ||
| .buildSchema(); | ||
| final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema) | ||
| .addSingleCol(intArray(123, 124)) | ||
| .addSingleCol(intArray(234, 235)) | ||
| .build(); | ||
|
|
||
| // Compare | ||
|
|
||
| RowSetUtilities.verify(expected, actual); | ||
| } | ||
| } |
Oops, something went wrong.