diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd index 31eef193ec5..f4df301df24 100644 --- a/exec/java-exec/src/main/codegen/data/Casts.tdd +++ b/exec/java-exec/src/main/codegen/data/Casts.tdd @@ -168,5 +168,25 @@ {from: "NullableVarBinary", to: "NullableFloat8", major: "EmptyString", javaType:"Double", parse:"Double"}, {from: "NullableVarBinary", to: "NullableVarDecimal", major: "NullableVarCharDecimalComplex"}, + + {from: "UntypedNull", to: "TinyInt", major: "UntypedNull"}, + {from: "UntypedNull", to: "Int", major: "UntypedNull"}, + {from: "UntypedNull", to: "BigInt", major: "UntypedNull"}, + {from: "UntypedNull", to: "Float4", major: "UntypedNull"}, + {from: "UntypedNull", to: "Float8", major: "UntypedNull"}, + {from: "UntypedNull", to: "Date", major: "UntypedNull"}, + {from: "UntypedNull", to: "Time", major: "UntypedNull"}, + {from: "UntypedNull", to: "TimeStamp", major: "UntypedNull"}, + {from: "UntypedNull", to: "Interval", major: "UntypedNull"}, + {from: "UntypedNull", to: "IntervalDay", major: "UntypedNull"}, + {from: "UntypedNull", to: "IntervalYear", major: "UntypedNull"}, + {from: "UntypedNull", to: "VarBinary", major: "UntypedNull"}, + {from: "UntypedNull", to: "VarChar", major: "UntypedNull"}, + {from: "UntypedNull", to: "Var16Char", major: "UntypedNull"}, + {from: "UntypedNull", to: "VarDecimal", major: "UntypedNull"}, + {from: "UntypedNull", to: "Decimal9", major: "UntypedNull"}, + {from: "UntypedNull", to: "Decimal18", major: "UntypedNull"}, + {from: "UntypedNull", to: "Decimal28Sparse", major: "UntypedNull"}, + {from: "UntypedNull", to: "Decimal38Sparse", major: "UntypedNull"}, ] } diff --git a/exec/java-exec/src/main/codegen/templates/CastUntypedNull.java b/exec/java-exec/src/main/codegen/templates/CastUntypedNull.java new file mode 100644 index 00000000000..66b74e259e4 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/CastUntypedNull.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> + +<#list cast.types as type> +<#if type.major == "UntypedNull"> + +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}${type.to}.java" /> + +<#include "/@includes/license.ftl" /> +package org.apache.drill.exec.expr.fn.impl.gcast; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.vector.UntypedNullHolder; + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ +@FunctionTemplate(name = "cast${type.to?upper_case}", + scope = FunctionTemplate.FunctionScope.SIMPLE, + nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) +public class Cast${type.from}${type.to} implements DrillSimpleFunc { + + @Param ${type.from}Holder in; + <#if type.to.contains("Decimal")> + @Param IntHolder precision; + @Param IntHolder scale; + <#elseif type.to == "VarChar" || type.to == "VarBinary" || type.to == "Var16Char"> + @Param BigIntHolder len; + + @Output Nullable${type.to}Holder out; + + public void setup() { + } + + public void eval() { + out.isSet = 0; + } +} + <#-- type.major --> + + diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java index 696d6db27f4..cd3a22f6175 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java @@ -36,6 +36,7 @@ import org.apache.drill.exec.record.selection.SelectionVector2; import org.apache.drill.exec.record.selection.SelectionVector4; import org.apache.drill.exec.vector.AllocationHelper; +import org.apache.drill.exec.vector.UntypedNullVector; import org.apache.drill.exec.vector.ValueVector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,6 +129,11 @@ public boolean load(RecordBatchDef def, DrillBuf buf) throws SchemaChangeExcepti // Load the vector. if (buf == null) { + // Buffers for untyped null vectors are always null and for the case + // field value alone is sufficient to load the vector + if (vector instanceof UntypedNullVector) { + vector.load(field, null); + } // Schema only } else if (field.getValueCount() == 0) { AllocationHelper.allocate(vector, 0, 0, 0); diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUntypedNull.java b/exec/java-exec/src/test/java/org/apache/drill/TestUntypedNull.java index 4976947eaa2..5f6c297bc18 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestUntypedNull.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestUntypedNull.java @@ -106,5 +106,91 @@ public void testTypeAndMode() throws Exception { assertEquals(0, summary.recordCount()); } + @Test + public void testCoalesceOnNotExistentColumns() throws Exception { + String query = "select coalesce(unk1, unk2) as coal from cp.`tpch/nation.parquet` limit 5"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("coal") + .baselineValuesForSingleColumn(null, null, null, null, null) + .go(); + } + + @Test + public void testCoalesceOnNotExistentColumnsWithGroupBy() throws Exception { + String query = "select coalesce(unk1, unk2) as coal from cp.`tpch/nation.parquet` group by 1"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("coal") + .baselineValuesForSingleColumn(new Object[] {null}) + .go(); + } + + @Test + public void testCoalesceOnNotExistentColumnsWithOrderBy() throws Exception { + String query = "select coalesce(unk1, unk2) as coal from cp.`tpch/nation.parquet` order by 1 limit 5"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("coal") + .baselineValuesForSingleColumn(null, null, null, null, null) + .go(); + } + + @Test + public void testCoalesceOnNotExistentColumnsWithCoalesceInWhereClause() throws Exception { + String query = "select coalesce(unk1, unk2) as coal from cp.`tpch/nation.parquet` where coalesce(unk1, unk2) > 10"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .expectsNumRecords(0) + .go(); + } + + @Test + public void testCoalesceOnNotExistentColumnsWithCoalesceInHavingClause() throws Exception { + String query = "select 1 from cp.`tpch/nation.parquet` group by n_name having count(coalesce(unk1, unk2)) > 10"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .expectsNumRecords(0) + .go(); + } + + @Test + public void testPartitionByCoalesceOnNotExistentColumns() throws Exception { + String query = + "select row_number() over (partition by coalesce(unk1, unk2)) as row_num from cp.`tpch/nation.parquet` limit 5"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("row_num") + .baselineValuesForSingleColumn(1L, 2L, 3L, 4L, 5L) + .go(); + } + + @Test + public void testCoalesceOnNotExistentColumnsInUDF() throws Exception { + String query = "select substr(coalesce(unk1, unk2), 1, 2) as coal from cp.`tpch/nation.parquet` limit 5"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("coal") + .baselineValuesForSingleColumn(null, null, null, null, null) + .go(); + } + + @Test + public void testCoalesceOnNotExistentColumnsInUDF2() throws Exception { + String query = "select abs(coalesce(unk1, unk2)) as coal from cp.`tpch/nation.parquet` limit 5"; + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("coal") + .baselineValuesForSingleColumn(null, null, null, null, null) + .go(); + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java index 0d884b9319c..1e1c6d90bda 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java @@ -749,4 +749,21 @@ public void testCastTimeLiteralInFilter() throws Exception { run("drop table if exists dfs.tmp.test_time_filter"); } } + + @Test + public void testCastUntypedNull() throws Exception { + String[] types = new String[] { + "BOOLEAN", "INT", "BIGINT", "FLOAT", "DOUBLE", "DATE", "TIME", "TIMESTAMP", "INTERVAL MONTH", + "INTERVAL YEAR", "VARBINARY", "VARCHAR", "DECIMAL(9)", "DECIMAL(18)", "DECIMAL(28)", "DECIMAL(38)" + }; + String query = "select cast(coalesce(unk1, unk2) as %s) as coal from cp.`tpch/nation.parquet` limit 1"; + for (String type : types) { + testBuilder() + .sqlQuery(String.format(query, type)) + .unOrdered() + .baselineColumns("coal") + .baselineValues(new Object[] {null}) + .go(); + } + } } diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillCursor.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillCursor.java index 7025f18608a..c8b8c5ede06 100644 --- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillCursor.java +++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillCursor.java @@ -470,12 +470,11 @@ private boolean nextRowInternally() throws SQLException { QueryDataBatch qrb = resultsListener.getNext(); // (Apparently:) Skip any spurious empty batches (batches that have - // zero rows and/or null data, other than the first batch (which carries + // zero rows and null data, other than the first batch (which carries // the (initial) schema but no rows)). - if ( afterFirstBatch ) { - while ( qrb != null - && ( qrb.getHeader().getRowCount() == 0 - || qrb.getData() == null ) ) { + if (afterFirstBatch) { + while (qrb != null + && (qrb.getHeader().getRowCount() == 0 && qrb.getData() == null)) { // Empty message--dispose of and try to get another. logger.warn( "Spurious batch read: {}", qrb ); diff --git a/exec/vector/src/main/codegen/templates/AbstractFieldReader.java b/exec/vector/src/main/codegen/templates/AbstractFieldReader.java index 4a763c211eb..7400d397e17 100644 --- a/exec/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/exec/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -29,9 +29,9 @@ * This class is generated using freemarker and the ${.template_name} template. */ @SuppressWarnings("unused") -abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader { +public abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader { - AbstractFieldReader() { + public AbstractFieldReader() { } /** diff --git a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java index 430a41b431b..f73438c157b 100644 --- a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java +++ b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java @@ -64,6 +64,8 @@ public static int getSize(MajorType major) { case FIXEDCHAR: return major.getPrecision(); case FIXED16CHAR: return major.getPrecision(); case FIXEDBINARY: return major.getPrecision(); + case NULL: + return 0; } throw new UnsupportedOperationException(buildErrorMessage("get size", major)); } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedHolderReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedHolderReaderImpl.java similarity index 92% rename from exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedHolderReaderImpl.java rename to exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedHolderReaderImpl.java index 3e10d7848c3..66f03f1f7e2 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedHolderReaderImpl.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedHolderReaderImpl.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.vector.complex.impl; +package org.apache.drill.exec.vector; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.vector.UntypedNullHolder; +import org.apache.drill.exec.vector.complex.impl.AbstractFieldReader; public class UntypedHolderReaderImpl extends AbstractFieldReader { @@ -47,5 +47,4 @@ public TypeProtos.MajorType getType() { public boolean isSet() { return false; } - } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedNullVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedNullVector.java index 9dd8480cb9a..394aa75779d 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedNullVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedNullVector.java @@ -17,8 +17,6 @@ */ package org.apache.drill.exec.vector; - -import org.apache.drill.exec.vector.complex.impl.UntypedReaderImpl; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; import io.netty.buffer.DrillBuf; import org.apache.drill.exec.memory.BufferAllocator; @@ -47,7 +45,6 @@ public final class UntypedNullVector extends BaseDataValueVector implements Fixe public UntypedNullVector(MaterializedField field, BufferAllocator allocator) { super(field, allocator); - valueCount = 0; } @Override @@ -77,7 +74,9 @@ public void allocateNew() { } public boolean allocateNewSafe() { return true; } @Override - public void allocateNew(final int valueCount) { } + public void allocateNew(final int valueCount) { + this.valueCount = valueCount; + } @Override public void reset() { } @@ -125,7 +124,9 @@ public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((UntypedNullVector) to); } - public void transferTo(UntypedNullVector target) { } + public void transferTo(UntypedNullVector target) { + target.valueCount = valueCount; + } public void splitAndTransferTo(int startIndex, int length, UntypedNullVector target) { } @@ -170,7 +171,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, UntypedNullVector from) { @Override public void copyEntry(int toIndex, ValueVector from, int fromIndex) { - ((UntypedNullVector) from).data.getBytes(fromIndex * 4, data, toIndex * 4, 4); } public final class Accessor extends BaseAccessor { diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedReader.java similarity index 90% rename from exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedReader.java rename to exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedReader.java index f15b8520e91..ec24f2cb25b 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedReader.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedReader.java @@ -15,9 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.vector.complex.impl; +package org.apache.drill.exec.vector; -import org.apache.drill.exec.vector.UntypedNullHolder; import org.apache.drill.exec.vector.complex.reader.BaseReader; public interface UntypedReader extends BaseReader { @@ -26,5 +25,4 @@ public interface UntypedReader extends BaseReader { int size(); void read(UntypedNullHolder holder); void read(int arrayIndex, UntypedNullHolder holder); - } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedReaderImpl.java similarity index 92% rename from exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedReaderImpl.java rename to exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedReaderImpl.java index da6f63ed212..924156e0d2c 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UntypedReaderImpl.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/UntypedReaderImpl.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.vector.complex.impl; +package org.apache.drill.exec.vector; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.vector.UntypedNullHolder; +import org.apache.drill.exec.vector.complex.impl.AbstractFieldReader; public class UntypedReaderImpl extends AbstractFieldReader { @@ -46,5 +46,4 @@ public void read(UntypedNullHolder holder) { public void read(int arrayIndex, UntypedNullHolder holder) { holder.isSet = 0; } - } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java index 488306ec021..de165d0e783 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java @@ -17,14 +17,12 @@ */ package org.apache.drill.exec.vector.complex.reader; -import org.apache.drill.exec.vector.complex.impl.UntypedReader; +import org.apache.drill.exec.vector.UntypedReader; import org.apache.drill.exec.vector.complex.reader.BaseReader.ListReader; import org.apache.drill.exec.vector.complex.reader.BaseReader.MapReader; import org.apache.drill.exec.vector.complex.reader.BaseReader.RepeatedListReader; import org.apache.drill.exec.vector.complex.reader.BaseReader.RepeatedMapReader; import org.apache.drill.exec.vector.complex.reader.BaseReader.ScalarReader; - - public interface FieldReader extends MapReader, ListReader, ScalarReader, RepeatedMapReader, RepeatedListReader, UntypedReader { -} \ No newline at end of file +}