From e2e8898f7dfca8f0b58a7135b5485ee45c273e47 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Thu, 1 Oct 2015 03:26:34 -0700 Subject: [PATCH 1/5] DRILL-3229: Implement Union type vector --- .../common/expression/fn/CastFunctions.java | 1 + .../org/apache/drill/common/types/Types.java | 5 + .../templates/AbstractFieldReader.java | 4 + .../templates/AbstractFieldWriter.java | 10 + .../templates/AbstractRecordWriter.java | 5 + .../main/codegen/templates/BaseReader.java | 3 + .../main/codegen/templates/BaseWriter.java | 4 +- .../main/codegen/templates/ComplexCopier.java | 134 ++++++ .../codegen/templates/ComplexReaders.java | 9 +- .../templates/EventBasedRecordWriter.java | 4 + .../codegen/templates/HolderReaderImpl.java | 22 + .../main/codegen/templates/ListWriters.java | 8 +- .../main/codegen/templates/MapWriters.java | 46 +- .../main/codegen/templates/NullReader.java | 10 +- .../main/codegen/templates/RecordWriter.java | 1 + .../main/codegen/templates/TypeHelper.java | 25 +- .../codegen/templates/UnionFunctions.java | 88 ++++ .../codegen/templates/UnionListWriter.java | 187 ++++++++ .../main/codegen/templates/UnionReader.java | 183 ++++++++ .../main/codegen/templates/UnionVector.java | 433 ++++++++++++++++++ .../main/codegen/templates/UnionWriter.java | 228 +++++++++ .../org/apache/drill/exec/ExecConstants.java | 2 + .../exec/expr/ExpressionTreeMaterializer.java | 83 ++-- .../drill/exec/expr/GetSetVectorHelper.java | 4 + .../drill/exec/expr/fn/DrillFuncHolder.java | 2 +- .../exec/expr/fn/impl/MappifyUtility.java | 4 +- .../exec/expr/fn/impl/UnionFunctions.java | 115 +++++ .../drill/exec/expr/holders/UnionHolder.java | 37 ++ .../impl/filter/FilterRecordBatch.java | 4 +- .../impl/flatten/FlattenRecordBatch.java | 4 +- .../impl/project/ProjectRecordBatch.java | 5 +- .../exec/record/AbstractRecordBatch.java | 9 + .../exec/record/SimpleVectorWrapper.java | 11 + .../exec/resolver/ResolverTypePrecedence.java | 1 + .../drill/exec/resolver/TypeCastRules.java | 10 + .../server/options/SystemOptionManager.java | 1 + .../exec/store/avro/MapOrListWriter.java | 4 +- .../store/easy/json/JSONRecordReader.java | 4 +- .../store/easy/json/JsonRecordWriter.java | 24 + .../vector/accessor/UnionSqlAccessor.java | 129 ++++++ .../drill/exec/vector/complex/ListVector.java | 292 ++++++++++++ .../exec/vector/complex/fn/JsonReader.java | 147 +++--- .../exec/vector/complex/fn/JsonWriter.java | 11 +- .../complex/impl/AbstractBaseReader.java | 17 + .../complex/impl/ComplexWriterImpl.java | 12 +- .../vector/complex/impl/UnionListReader.java | 90 ++++ .../complex/impl/VectorContainerWriter.java | 8 +- .../drill/exec/store/TestOutputMutator.java | 4 + .../vector/complex/writer/TestJsonReader.java | 104 +++++ .../vector/complex/writer/TestRepeated.java | 24 +- .../src/test/resources/jsoninput/union/a.json | 52 +++ .../drill/jdbc/test/TestJdbcDistQuery.java | 1 - .../apache/drill/common/types/MinorType.java | 4 +- .../apache/drill/common/types/TypeProtos.java | 18 +- protocol/src/main/protobuf/Types.proto | 1 + 55 files changed, 2492 insertions(+), 156 deletions(-) create mode 100644 exec/java-exec/src/main/codegen/templates/ComplexCopier.java create mode 100644 exec/java-exec/src/main/codegen/templates/UnionFunctions.java create mode 100644 exec/java-exec/src/main/codegen/templates/UnionListWriter.java create mode 100644 exec/java-exec/src/main/codegen/templates/UnionReader.java create mode 100644 exec/java-exec/src/main/codegen/templates/UnionVector.java create mode 100644 exec/java-exec/src/main/codegen/templates/UnionWriter.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java create mode 100644 exec/java-exec/src/test/resources/jsoninput/union/a.json diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java index c0eaa9042e9..34997ab796d 100644 --- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java +++ b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java @@ -36,6 +36,7 @@ public class CastFunctions { private static Map CAST_FUNC_REPLACEMENT_FROM_NULLABLE = new HashMap<>(); static { + TYPE2FUNC.put(MinorType.UNION, "castUNION"); TYPE2FUNC.put(MinorType.BIGINT, "castBIGINT"); TYPE2FUNC.put(MinorType.INT, "castINT"); TYPE2FUNC.put(MinorType.BIT, "castBIT"); diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java index 69b1b4c72d0..adc3eef49e2 100644 --- a/common/src/main/java/org/apache/drill/common/types/Types.java +++ b/common/src/main/java/org/apache/drill/common/types/Types.java @@ -143,6 +143,7 @@ public static String getSqlTypeName(final MajorType type) { case MAP: return "MAP"; case LATE: return "ANY"; case NULL: return "NULL"; + case UNION: return "UNION"; // Internal types not actually used at level of SQL types(?): @@ -228,6 +229,8 @@ public static int getJdbcTypeCode(final MajorType type) { return java.sql.Types.VARBINARY; case VARCHAR: return java.sql.Types.VARCHAR; + case UNION: + return java.sql.Types.OTHER; default: // TODO: This isn't really an unsupported-operation/-type case; this // is an unexpected, code-out-of-sync-with-itself case, so use an @@ -290,6 +293,7 @@ public static boolean isJdbcSignedType( final MajorType type ) { case LATE: case LIST: case MAP: + case UNION: case NULL: case TIMETZ: // SQL TIME WITH TIME ZONE case TIMESTAMPTZ: // SQL TIMESTAMP WITH TIME ZONE @@ -340,6 +344,7 @@ public static boolean isFixedWidthType(final MajorType type) { case VARBINARY: case VAR16CHAR: case VARCHAR: + case UNION: return false; default: return true; diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java index 5420f998434..89afd7c5505 100644 --- a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java +++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java @@ -67,6 +67,10 @@ public void copyAsField(String name, ListWriter writer){ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign boxedType = (minor.boxedType!type.boxedType) /> + public void read(${name}Holder holder){ + fail("${name}"); + } + public void read(Nullable${name}Holder holder){ fail("${name}"); } diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java index 9b673048e56..2da714111ce 100644 --- a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java +++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java @@ -45,6 +45,16 @@ public void end() { throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); } + @Override + public void startList() { + throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); + } + + @Override + public void endList() { + throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); + } + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> @Override diff --git a/exec/java-exec/src/main/codegen/templates/AbstractRecordWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractRecordWriter.java index 5f1f42f239d..13f74827c0b 100644 --- a/exec/java-exec/src/main/codegen/templates/AbstractRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/AbstractRecordWriter.java @@ -54,6 +54,11 @@ public FieldConverter getNewMapConverter(int fieldId, String fieldName, FieldRea throw new UnsupportedOperationException("Doesn't support writing Map'"); } + @Override + public FieldConverter getNewUnionConverter(int fieldId, String fieldName, FieldReader reader) { + throw new UnsupportedOperationException("Doesn't support writing Union type'"); + } + @Override public FieldConverter getNewRepeatedMapConverter(int fieldId, String fieldName, FieldReader reader) { throw new UnsupportedOperationException("Doesn't support writing RepeatedMap"); diff --git a/exec/java-exec/src/main/codegen/templates/BaseReader.java b/exec/java-exec/src/main/codegen/templates/BaseReader.java index 116deda8960..4fce409ecdd 100644 --- a/exec/java-exec/src/main/codegen/templates/BaseReader.java +++ b/exec/java-exec/src/main/codegen/templates/BaseReader.java @@ -33,6 +33,9 @@ public interface BaseReader extends Positionable{ MajorType getType(); MaterializedField getField(); void reset(); + void read(UnionHolder holder); + void read(int index, UnionHolder holder); + void copyAsValue(UnionWriter writer); public interface MapReader extends BaseReader, Iterable{ FieldReader reader(String name); diff --git a/exec/java-exec/src/main/codegen/templates/BaseWriter.java b/exec/java-exec/src/main/codegen/templates/BaseWriter.java index 76978801fb3..da27e660f6f 100644 --- a/exec/java-exec/src/main/codegen/templates/BaseWriter.java +++ b/exec/java-exec/src/main/codegen/templates/BaseWriter.java @@ -57,8 +57,8 @@ public interface MapWriter extends BaseWriter { } public interface ListWriter extends BaseWriter { - void start(); - void end(); + void startList(); + void endList(); MapWriter map(); ListWriter list(); void copyReader(FieldReader reader); diff --git a/exec/java-exec/src/main/codegen/templates/ComplexCopier.java b/exec/java-exec/src/main/codegen/templates/ComplexCopier.java new file mode 100644 index 00000000000..c3b5cb56b0e --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/ComplexCopier.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/ComplexCopier.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ +@SuppressWarnings("unused") +public class ComplexCopier { + + FieldReader in; + FieldWriter out; + + public ComplexCopier(FieldReader in, FieldWriter out) { + this.in = in; + this.out = out; + } + + public void write() { + writeValue(in, out); + } + + private void writeValue(FieldReader reader, FieldWriter writer) { + final DataMode m = reader.getType().getMode(); + final MinorType mt = reader.getType().getMinorType(); + + switch(m){ + case OPTIONAL: + case REQUIRED: + + + switch (mt) { + + case LIST: + writer.startList(); + while (reader.next()) { + writeValue(reader.reader(), getListWriterForReader(reader.reader(), writer)); + } + writer.endList(); + break; + case MAP: + writer.start(); + if (reader.isSet()) { + for(String name : reader){ + FieldReader childReader = reader.reader(name); + if(childReader.isSet()){ + writeValue(childReader, getMapWriterForReader(childReader, writer, name)); + } + } + } + writer.end(); + break; + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + + case ${name?upper_case}: + if (reader.isSet()) { + Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder(); + reader.read(${uncappedName}Holder); + writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, ); + } + break; + + + + } + break; + } + } + + private FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) { + switch (reader.getType().getMinorType()) { + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + case ${name?upper_case}: + return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}(name); + + + case MAP: + return (FieldWriter) writer.map(name); + case LIST: + return (FieldWriter) writer.list(name); + default: + throw new UnsupportedOperationException(reader.getType().toString()); + } + } + + private FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) { + switch (reader.getType().getMinorType()) { + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + case ${name?upper_case}: + return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}(); + + + case MAP: + return (FieldWriter) writer.map(); + case LIST: + return (FieldWriter) writer.list(); + default: + throw new UnsupportedOperationException(reader.getType().toString()); + } + } +} diff --git a/exec/java-exec/src/main/codegen/templates/ComplexReaders.java b/exec/java-exec/src/main/codegen/templates/ComplexReaders.java index 068efb4441e..607b71d94c0 100644 --- a/exec/java-exec/src/main/codegen/templates/ComplexReaders.java +++ b/exec/java-exec/src/main/codegen/templates/ComplexReaders.java @@ -119,7 +119,13 @@ public void copyAsField(String name, MapWriter writer){ ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer.${lowerName}(name); impl.vector.copyFromSafe(idx(), impl.idx(), vector); } - + + <#if nullMode != "Nullable"> + public void read(${minor.class?cap_first}Holder h){ + vector.getAccessor().get(idx(), h); + } + + public void read(Nullable${minor.class?cap_first}Holder h){ vector.getAccessor().get(idx(), h); } @@ -157,6 +163,7 @@ public interface ${name}Reader extends BaseReader{ public Object readObject(int arrayIndex); public ${friendlyType} read${safeType}(int arrayIndex); <#else> + public void read(${minor.class?cap_first}Holder h); public void read(Nullable${minor.class?cap_first}Holder h); public Object readObject(); public ${friendlyType} read${safeType}(); diff --git a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java index cf1529d215f..bf447c9c91a 100644 --- a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java @@ -31,6 +31,7 @@ import org.apache.drill.exec.planner.physical.WriterPrel; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.vector.complex.impl.UnionReader; import org.apache.drill.exec.vector.complex.reader.FieldReader; import java.io.IOException; @@ -118,6 +119,9 @@ public void endField() throws IOException { } public static FieldConverter getConverter(RecordWriter recordWriter, int fieldId, String fieldName, FieldReader reader) { + if (reader instanceof UnionReader) { + return recordWriter.getNewUnionConverter(fieldId, fieldName, reader); + } switch (reader.getType().getMinorType()) { case MAP: switch (reader.getType().getMode()) { diff --git a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java index 3bf94036c4e..9bb4f82dccd 100644 --- a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java +++ b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java @@ -30,6 +30,7 @@ <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /> +<#assign fields = minor.fields!type.fields /> <@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" /> <#include "/@includes/license.ftl" /> @@ -116,6 +117,22 @@ public boolean isSet() { } +<#if holderMode != "Repeated"> +@Override + public void read(${name}Holder h) { + <#list fields as field> + h.${field.name} = holder.${field.name}; + + } + + @Override + public void read(Nullable${name}Holder h) { + <#list fields as field> + h.${field.name} = holder.${field.name}; + + } + + <#if holderMode == "Repeated"> @Override public ${friendlyType} read${safeType}(int index){ @@ -262,6 +279,11 @@ private Object readSingleObject() { } +<#if holderMode != "Repeated" && nullMode != "Nullable"> + public void copyAsValue(${minor.class?cap_first}Writer writer){ + writer.write(holder); + } + } diff --git a/exec/java-exec/src/main/codegen/templates/ListWriters.java b/exec/java-exec/src/main/codegen/templates/ListWriters.java index f2683a3f024..16d41ecf1b8 100644 --- a/exec/java-exec/src/main/codegen/templates/ListWriters.java +++ b/exec/java-exec/src/main/codegen/templates/ListWriters.java @@ -181,7 +181,7 @@ public MaterializedField getField() { <#if mode == "Repeated"> - public void start() { + public void startList() { final RepeatedListVector list = (RepeatedListVector) container; final RepeatedListVector.RepeatedMutator mutator = list.getMutator(); @@ -202,7 +202,7 @@ public void start() { } } - public void end() { + public void endList() { // noop, we initialize state at start rather than end. } <#else> @@ -214,11 +214,11 @@ public void setPosition(int index) { } } - public void start() { + public void startList() { // noop } - public void end() { + public void endList() { // noop } diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java index 968333897c2..42cacabd3ee 100644 --- a/exec/java-exec/src/main/codegen/templates/MapWriters.java +++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java @@ -52,9 +52,18 @@ public class ${mode}MapWriter extends AbstractFieldWriter { private final Map fields = Maps.newHashMap(); <#if mode == "Repeated">private int currentChildIndex = 0; - public ${mode}MapWriter(${containerClass} container, FieldWriter parent) { + private final boolean unionEnabled; + private final boolean unionInternalMap; + + public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled, boolean unionInternalMap) { super(parent); this.container = container; + this.unionEnabled = unionEnabled; + this.unionInternalMap = unionInternalMap; + } + + public ${mode}MapWriter(${containerClass} container, FieldWriter parent) { + this(container, parent, false, false); } @Override @@ -70,10 +79,15 @@ public MaterializedField getField() { @Override public MapWriter map(String name) { FieldWriter writer = fields.get(name.toLowerCase()); - if(writer == null) { - int vectorCount = container.size(); - MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class); - writer = new SingleMapWriter(vector, this); + if(writer == null){ + int vectorCount=container.size(); + if(!unionEnabled || unionInternalMap){ + MapVector vector=container.addOrGet(name,MapVector.TYPE,MapVector.class); + writer=new SingleMapWriter(vector,this); + } else { + UnionVector vector = container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class); + writer = new UnionWriter(vector); + } if(vectorCount != container.size()) { writer.allocate(); } @@ -108,8 +122,16 @@ public void clear() { @Override public ListWriter list(String name) { FieldWriter writer = fields.get(name.toLowerCase()); + int vectorCount = container.size(); if(writer == null) { - writer = new SingleListWriter(name, container, this); + if (!unionEnabled){ + writer = new SingleListWriter(name,container,this); + } else{ + writer = new UnionWriter(container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class)); + } + if (container.size() > vectorCount) { + writer.allocate(); + } writer.setPosition(${index}); fields.put(name.toLowerCase(), writer); } @@ -191,9 +213,17 @@ public void end() { FieldWriter writer = fields.get(name.toLowerCase()); if(writer == null) { - final ${vectName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class); + ValueVector vector; + if (unionEnabled){ + UnionVector v = container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class); + writer = new UnionWriter(v); + vector = v; + } else { + ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class); + writer = new ${vectName}WriterImpl(v, this); + vector = v; + } vector.allocateNewSafe(); - writer = new ${vectName}WriterImpl(vector, this); writer.setPosition(${index}); fields.put(name.toLowerCase(), writer); } diff --git a/exec/java-exec/src/main/codegen/templates/NullReader.java b/exec/java-exec/src/main/codegen/templates/NullReader.java index a0e5f508f6e..472dbed557c 100644 --- a/exec/java-exec/src/main/codegen/templates/NullReader.java +++ b/exec/java-exec/src/main/codegen/templates/NullReader.java @@ -56,11 +56,17 @@ public void copyAsValue(MapWriter writer) {} public void copyAsValue(ListWriter writer) {} - <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + public void copyAsValue(UnionWriter writer) {} + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + public void read(${name}Holder holder){ + throw new UnsupportedOperationException("NullReader cannot read into non-nullable holder"); + } + public void read(Nullable${name}Holder holder){ holder.isSet = 0; } - + public void read(int arrayIndex, ${name}Holder holder){ throw new ArrayIndexOutOfBoundsException(); } diff --git a/exec/java-exec/src/main/codegen/templates/RecordWriter.java b/exec/java-exec/src/main/codegen/templates/RecordWriter.java index a37ffa88916..24a94c4c56f 100644 --- a/exec/java-exec/src/main/codegen/templates/RecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/RecordWriter.java @@ -62,6 +62,7 @@ public interface RecordWriter { /** Add the field value given in valueHolder at the given column number fieldId. */ public FieldConverter getNewMapConverter(int fieldId, String fieldName, FieldReader reader); + public FieldConverter getNewUnionConverter(int fieldId, String fieldName, FieldReader reader); public FieldConverter getNewRepeatedMapConverter(int fieldId, String fieldName, FieldReader reader); public FieldConverter getNewRepeatedListConverter(int fieldId, String fieldName, FieldReader reader); diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java index 9c66cb718dd..d9810c79ec9 100644 --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java @@ -16,6 +16,8 @@ * limitations under the License. */ +import org.apache.drill.exec.vector.complex.UnionVector; + <@pp.dropOutputFile /> <@pp.changeOutputFile name="/org/apache/drill/exec/expr/TypeHelper.java" /> @@ -71,6 +73,8 @@ public static int getSize(MajorType major) { public static SqlAccessor getSqlAccessor(ValueVector vector){ final MajorType type = vector.getField().getType(); switch(type.getMinorType()){ + case UNION: + return new UnionSqlAccessor((UnionVector) vector); <#list vv.types as type> <#list type.minor as minor> case ${minor.class?upper_case}: @@ -100,8 +104,11 @@ public static ValueVector getNewVector(SchemaPath parentPath, String name, Buffe public static Class getValueVectorClass(MinorType type, DataMode mode){ switch (type) { + case UNION: + return UnionVector.class; case MAP: switch (mode) { + case OPTIONAL: case REQUIRED: return MapVector.class; case REPEATED: @@ -112,6 +119,9 @@ public static Class getValueVectorClass(MinorType type, DataMode mode){ switch (mode) { case REPEATED: return RepeatedListVector.class; + case REQUIRED: + case OPTIONAL: + return ListVector.class; } <#list vv.types as type> @@ -175,6 +185,7 @@ public static Class getReaderClassName( MinorType type, DataMode mode, boolea public static Class getWriterInterface( MinorType type, DataMode mode){ switch (type) { + case UNION: return UnionWriter.class; case MAP: return MapWriter.class; case LIST: return ListWriter.class; <#list vv.types as type> @@ -190,6 +201,8 @@ public static Class getWriterInterface( MinorType type, DataMode mode){ public static Class getWriterImpl( MinorType type, DataMode mode){ switch (type) { + case UNION: + return UnionWriter.class; case MAP: switch (mode) { case REQUIRED: @@ -247,6 +260,8 @@ public static Class getHolderReaderImpl( MinorType type, DataMode mode){ public static JType getHolderType(JCodeModel model, MinorType type, DataMode mode){ switch (type) { + case UNION: + return model._ref(UnionHolder.class); case MAP: case LIST: return model._ref(ComplexHolder.class); @@ -280,10 +295,13 @@ public static ValueVector getNewVector(MaterializedField field, BufferAllocator switch (type.getMinorType()) { - + case UNION: + return new UnionVector(field, allocator, callBack); + case MAP: switch (type.getMode()) { case REQUIRED: + case OPTIONAL: return new MapVector(field, allocator, callBack); case REPEATED: return new RepeatedMapVector(field, allocator, callBack); @@ -292,7 +310,10 @@ public static ValueVector getNewVector(MaterializedField field, BufferAllocator switch (type.getMode()) { case REPEATED: return new RepeatedListVector(field, allocator, callBack); - } + case OPTIONAL: + case REQUIRED: + return new ListVector(field, allocator, callBack); + } <#list vv. types as type> <#list type.minor as minor> case ${minor.class?upper_case}: diff --git a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java new file mode 100644 index 00000000000..41b6b002fa8 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionFunctions.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.*; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; +import org.apache.drill.exec.record.RecordBatch; + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ + +@SuppressWarnings("unused") +public class UnionFunctions { + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + + <#if !minor.class?starts_with("Decimal")> + + @SuppressWarnings("unused") + @FunctionTemplate(name = "as${name}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class CastUnion${name} implements DrillSimpleFunc { + + @Param UnionHolder in; + @Output Nullable${name}Holder out; + + public void setup() {} + + public void eval() { + if (in.isSet == 1) { + in.reader.read(out); + } else { + out.isSet = 0; + } + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class Cast${name}ToUnion implements DrillSimpleFunc { + + @Param Nullable${name}Holder in; + @Output UnionHolder out; + + public void setup() {} + + public void eval() { + out.reader = new org.apache.drill.exec.vector.complex.impl.Nullable${name}HolderReaderImpl(in); + out.isSet = in.isSet; + } + } + + + + + +} diff --git a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java new file mode 100644 index 00000000000..fd7256b06ac --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionListWriter.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ + +@SuppressWarnings("unused") +public class UnionListWriter extends AbstractFieldWriter { + + ListVector vector; + UnionVector data; + UInt4Vector offsets; + private UnionWriter writer; + private boolean inMap = false; + private String mapName; + private int lastIndex = 0; + + public UnionListWriter(ListVector vector) { + super(null); + this.vector = vector; + this.data = (UnionVector) vector.getDataVector(); + this.writer = new UnionWriter(data); + this.offsets = vector.getOffsetVector(); + } + + @Override + public void allocate() { + vector.allocateNew(); + } + + @Override + public void clear() { + vector.clear(); + } + + @Override + public MaterializedField getField() { + return null; + } + + @Override + public int getValueCapacity() { + return vector.getValueCapacity(); + } + + @Override + public void close() throws Exception { + + } + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + + <#if !minor.class?starts_with("Decimal")> + + @Override + public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}() { + return this; + } + + @Override + public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}(String name) { + assert inMap; + mapName = name; + return this; + } + + + + + + @Override + public MapWriter map() { + inMap = true; + return this; + } + + @Override + public ListWriter list() { + final int nextOffset = offsets.getAccessor().get(idx() + 1); + vector.getMutator().setNotNull(idx()); + offsets.getMutator().setSafe(idx() + 1, nextOffset + 1); + writer.setPosition(nextOffset); + return writer; + } + + @Override + public ListWriter list(String name) { + final int nextOffset = offsets.getAccessor().get(idx() + 1); + vector.getMutator().setNotNull(idx()); + data.getMutator().setType(nextOffset, MinorType.MAP); + writer.setPosition(nextOffset); + ListWriter listWriter = writer.list(name); + return listWriter; + } + + @Override + public MapWriter map(String name) { + MapWriter mapWriter = writer.map(name); + return mapWriter; + } + + @Override + public void startList() { + vector.getMutator().startNewValue(idx()); + } + + @Override + public void endList() { + + } + + @Override + public void start() { + assert inMap; + final int nextOffset = offsets.getAccessor().get(idx() + 1); + vector.getMutator().setNotNull(idx()); + data.getMutator().setType(nextOffset, MinorType.MAP); + offsets.getMutator().setSafe(idx() + 1, nextOffset); + writer.setPosition(nextOffset); + } + + @Override + public void end() { + if (inMap) { + inMap = false; + final int nextOffset = offsets.getAccessor().get(idx() + 1); + offsets.getMutator().setSafe(idx() + 1, nextOffset + 1); + } + } + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + + <#if !minor.class?starts_with("Decimal")> + + @Override + public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { + if (inMap) { + final int nextOffset = offsets.getAccessor().get(idx() + 1); + vector.getMutator().setNotNull(idx()); + data.getMutator().setType(nextOffset, MinorType.MAP); + writer.setPosition(nextOffset); + ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}(mapName); + ${uncappedName}Writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); + } else { + final int nextOffset = offsets.getAccessor().get(idx() + 1); + vector.getMutator().setNotNull(idx()); + writer.setPosition(nextOffset); + writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); + offsets.getMutator().setSafe(idx() + 1, nextOffset + 1); + } + } + + + + + +} diff --git a/exec/java-exec/src/main/codegen/templates/UnionReader.java b/exec/java-exec/src/main/codegen/templates/UnionReader.java new file mode 100644 index 00000000000..38d424786e9 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/UnionReader.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.vector.complex.impl.NullReader; + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionReader.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> + +@SuppressWarnings("unused") +public class UnionReader extends AbstractFieldReader { + + private BaseReader[] readers = new BaseReader[43]; + public UnionVector data; + + public UnionReader(UnionVector data) { + this.data = data; + } + + public MajorType getType() { + return Types.required(MinorType.valueOf(data.getTypeValue(idx()))); + } + + public boolean isSet(){ + return !data.getAccessor().isNull(idx()); + } + + public void read(UnionHolder holder) { + holder.reader = this; + holder.isSet = this.isSet() ? 1 : 0; + } + + public void read(int index, UnionHolder holder) { + getList().read(index, holder); + } + + private FieldReader getReaderForIndex(int index) { + int typeValue = data.getTypeValue(index); + FieldReader reader = (FieldReader) readers[typeValue]; + if (reader != null) { + return reader; + } + switch (typeValue) { + case 0: + return NullReader.INSTANCE; + case MinorType.MAP_VALUE: + return (FieldReader) getMap(); + case MinorType.LIST_VALUE: + return (FieldReader) getList(); + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + case MinorType.${name?upper_case}_VALUE: + return (FieldReader) get${name}(); + + + default: + throw new UnsupportedOperationException("Unsupported type: " + MinorType.valueOf(typeValue)); + } + } + + private SingleMapReaderImpl mapReader; + + private MapReader getMap() { + if (mapReader == null) { + mapReader = (SingleMapReaderImpl) data.getMap().getReader(); + mapReader.setPosition(idx()); + readers[MinorType.MAP_VALUE] = mapReader; + } + return mapReader; + } + + private UnionListReader listReader; + + private FieldReader getList() { + if (listReader == null) { + listReader = new UnionListReader(data.getList()); + listReader.setPosition(idx()); + readers[MinorType.LIST_VALUE] = listReader; + } + return listReader; + } + + @Override + public java.util.Iterator iterator() { + return getMap().iterator(); + } + + @Override + public void copyAsValue(UnionWriter writer) { + writer.data.copyFrom(idx(), writer.idx(), data); + } + + <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", + "Character", "DateTime", "Period", "Double", "Float", + "Text", "String", "Byte", "Short", "byte[]"] as friendlyType> + <#assign safeType=friendlyType /> + <#if safeType=="byte[]"><#assign safeType="ByteArray" /> + + @Override + public ${friendlyType} read${safeType}() { + return getReaderForIndex(idx()).read${safeType}(); + } + + + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign uncappedName = name?uncap_first/> + <#assign boxedType = (minor.boxedType!type.boxedType) /> + <#assign javaType = (minor.javaType!type.javaType) /> + <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> + <#assign safeType=friendlyType /> + <#if safeType=="byte[]"><#assign safeType="ByteArray" /> + <#if !minor.class?starts_with("Decimal")> + + private Nullable${name}ReaderImpl ${uncappedName}Reader; + + private Nullable${name}ReaderImpl get${name}() { + if (${uncappedName}Reader == null) { + ${uncappedName}Reader = new Nullable${name}ReaderImpl(data.get${name}Vector()); + ${uncappedName}Reader.setPosition(idx()); + readers[MinorType.${name?upper_case}_VALUE] = ${uncappedName}Reader; + } + return ${uncappedName}Reader; + } + + public void read(Nullable${name}Holder holder){ + getReaderForIndex(idx()).read(holder); + } + + public void copyAsValue(${name}Writer writer){ + getReaderForIndex(idx()).copyAsValue(writer); + } + + + + @Override + public void setPosition(int index) { + super.setPosition(index); + for (BaseReader reader : readers) { + if (reader != null) { + reader.setPosition(index); + } + } + } + + public FieldReader reader(String name){ + return getMap().reader(name); + } + + public FieldReader reader() { + return getList().reader(); + } + + public boolean next() { + return getReaderForIndex(idx()).next(); + } +} + + + diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java new file mode 100644 index 00000000000..6a72757cc85 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/UnionVector.java @@ -0,0 +1,433 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.drill.common.types.TypeProtos.MinorType; + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionVector.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> +import java.util.Iterator; +import org.apache.drill.exec.vector.complex.impl.ComplexCopier; +import org.apache.drill.exec.util.CallBack; + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ +@SuppressWarnings("unused") + + +public class UnionVector implements ValueVector { + + private MaterializedField field; + private BufferAllocator allocator; + private Accessor accessor = new Accessor(); + private Mutator mutator = new Mutator(); + private int valueCount; + + private MapVector internalMap; + private SingleMapWriter internalMapWriter; + private UInt1Vector typeVector; + + private MapVector mapVector; + private ListVector listVector; + private NullableBigIntVector bigInt; + private NullableVarCharVector varChar; + + private FieldReader reader; + private NullableBitVector bit; + + private State state = State.INIT; + private int singleType = 0; + private ValueVector singleVector; + + private enum State { + INIT, SINGLE, MULTI + } + + public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { + this.field = field.clone(); + this.allocator = allocator; + internalMap = new MapVector("internal", allocator, callBack); + internalMapWriter = new SingleMapWriter(internalMap, null, true, true); + this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class); + this.field.addChild(internalMap.getField().clone()); + } + + private void updateState(ValueVector v) { + if (state == State.INIT) { + state = State.SINGLE; + singleVector = v; + singleType = v.getField().getType().getMinorType().getNumber(); + } else { + state = State.MULTI; + singleVector = null; + } + } + + public boolean isSingleType() { + return state == State.SINGLE && singleType != MinorType.LIST_VALUE; + } + + public ValueVector getSingleVector() { + assert state != State.MULTI : "Cannot get single vector when there are multiple types"; + assert state != State.INIT : "Cannot get single vector when there are no types"; + return singleVector; + } + + public MapVector getMap() { + if (mapVector == null) { + int vectorCount = internalMap.size(); + mapVector = internalMap.addOrGet("map", Types.optional(MinorType.MAP), MapVector.class); + updateState(mapVector); + if (internalMap.size() > vectorCount) { + mapVector.allocateNew(); + } + } + return mapVector; + } + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + + private Nullable${name}Vector ${uncappedName}Vector; + + public Nullable${name}Vector get${name}Vector() { + if (${uncappedName}Vector == null) { + int vectorCount = internalMap.size(); + ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", Types.optional(MinorType.${name?upper_case}), Nullable${name}Vector.class); + updateState(${uncappedName}Vector); + if (internalMap.size() > vectorCount) { + ${uncappedName}Vector.allocateNew(); + } + } + return ${uncappedName}Vector; + } + + + + + + public ListVector getList() { + if (listVector == null) { + int vectorCount = internalMap.size(); + listVector = internalMap.addOrGet("list", Types.optional(MinorType.LIST), ListVector.class); + updateState(listVector); + if (internalMap.size() > vectorCount) { + listVector.allocateNew(); + } + } + return listVector; + } + + public int getTypeValue(int index) { + return typeVector.getAccessor().get(index); + } + + public UInt1Vector getTypeVector() { + return typeVector; + } + + @Override + public void allocateNew() throws OutOfMemoryRuntimeException { + internalMap.allocateNew(); + if (typeVector != null) { + typeVector.zeroVector(); + } + } + + @Override + public boolean allocateNewSafe() { + boolean safe = internalMap.allocateNewSafe(); + if (safe) { + if (typeVector != null) { + typeVector.zeroVector(); + } + } + return safe; + } + + @Override + public void setInitialCapacity(int numRecords) { + } + + @Override + public int getValueCapacity() { + return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity()); + } + + @Override + public void close() { + } + + @Override + public void clear() { + internalMap.clear(); + } + + @Override + public MaterializedField getField() { + return field; + } + + @Override + public TransferPair getTransferPair() { + return new TransferImpl(field); + } + + @Override + public TransferPair getTransferPair(FieldReference ref) { + return new TransferImpl(field.withPath(ref)); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return new TransferImpl((UnionVector) target); + } + + public void transferTo(UnionVector target) { + internalMap.makeTransferPair(target.internalMap).transfer(); + target.valueCount = valueCount; + } + + public void copyFrom(int inIndex, int outIndex, UnionVector from) { + from.getReader().setPosition(inIndex); + getWriter().setPosition(outIndex); + ComplexCopier copier = new ComplexCopier(from.reader, mutator.writer); + copier.write(); + } + + public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { + copyFrom(inIndex, outIndex, from); + } + + private class TransferImpl implements TransferPair { + + UnionVector to; + + public TransferImpl(MaterializedField field) { + to = new UnionVector(field, allocator, null); + } + + public TransferImpl(UnionVector to) { + this.to = to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + + } + + @Override + public ValueVector getTo() { + return to; + } + + @Override + public void copyValueSafe(int from, int to) { + this.to.copyFrom(from, to, UnionVector.this); + } + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public FieldReader getReader() { + if (reader == null) { + reader = new UnionReader(this); + } + return reader; + } + + public FieldWriter getWriter() { + if (mutator.writer == null) { + mutator.writer = new UnionWriter(this); + } + return mutator.writer; + } + + @Override + public UserBitShared.SerializedField getMetadata() { + SerializedField.Builder b = getField() // + .getAsBuilder() // + .setBufferLength(getBufferSize()) // + .setValueCount(valueCount); + + b.addChild(internalMap.getMetadata()); + return b.build(); + } + + @Override + public int getBufferSize() { + return internalMap.getBufferSize(); + } + + @Override + public DrillBuf[] getBuffers(boolean clear) { + return internalMap.getBuffers(clear); + } + + @Override + public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { + valueCount = metadata.getValueCount(); + + internalMap.load(metadata.getChild(0), buffer); + } + + @Override + public Iterator iterator() { + return null; + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + + @Override + public Object getObject(int index) { + int type = typeVector.getAccessor().get(index); + switch (type) { + case 0: + return null; + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + case MinorType.${name?upper_case}_VALUE: + return get${name}Vector().getAccessor().getObject(index); + + + + case MinorType.MAP_VALUE: + return getMap().getAccessor().getObject(index); + case MinorType.LIST_VALUE: + return getList().getAccessor().getObject(index); + default: + throw new UnsupportedOperationException("Cannot support type: " + MinorType.valueOf(type)); + } + } + + public byte[] get(int index) { + return null; + } + + public void get(int index, ComplexHolder holder) { + } + + public void get(int index, UnionHolder holder) { + if (reader == null) { + reader = new UnionReader(UnionVector.this); + } + reader.setPosition(index); + holder.reader = reader; + } + + @Override + public int getValueCount() { + return valueCount; + } + + @Override + public boolean isNull(int index) { + return typeVector.getAccessor().get(index) == 0; + } + + public int isSet(int index) { + return isNull(index) ? 0 : 1; + } + } + + public class Mutator extends BaseValueVector.BaseMutator { + + UnionWriter writer; + + @Override + public void setValueCount(int valueCount) { + UnionVector.this.valueCount = valueCount; + internalMap.getMutator().setValueCount(valueCount); + } + + public void set(int index, byte[] bytes) { + } + + public void setSafe(int index, UnionHolder holder) { + FieldReader reader = holder.reader; + if (writer == null) { + writer = new UnionWriter(UnionVector.this); + } + writer.setPosition(index); + MinorType type = reader.getType().getMinorType(); + switch (type) { + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + case ${name?upper_case}: + Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder(); + reader.read(${uncappedName}Holder); + if (holder.isSet == 1) { + writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, ); + } + break; + + + case MAP: { + ComplexCopier copier = new ComplexCopier(reader, writer); + copier.write(); + break; + } + case LIST: { + ComplexCopier copier = new ComplexCopier(reader, writer); + copier.write(); + break; + } + default: + throw new UnsupportedOperationException(); + } + } + + public void setType(int index, MinorType type) { + typeVector.getMutator().setSafe(index, type.getNumber()); + } + + @Override + public void reset() { } + + @Override + public void generateTestData(int values) { } + } +} diff --git a/exec/java-exec/src/main/codegen/templates/UnionWriter.java b/exec/java-exec/src/main/codegen/templates/UnionWriter.java new file mode 100644 index 00000000000..9d779999c17 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/UnionWriter.java @@ -0,0 +1,228 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionWriter.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ +@SuppressWarnings("unused") +public class UnionWriter extends AbstractFieldWriter implements FieldWriter { + + UnionVector data; + private MapWriter mapWriter; + private UnionListWriter listWriter; + private List writers = Lists.newArrayList(); + + public UnionWriter(BufferAllocator allocator) { + super(null); + } + + public UnionWriter(UnionVector vector) { + super(null); + data = vector; + } + + public UnionWriter(UnionVector vector, FieldWriter parent) { + super(null); + data = vector; + } + + @Override + public void setPosition(int index) { + super.setPosition(index); + for (BaseWriter writer : writers) { + writer.setPosition(index); + } + } + + + @Override + public void start() { + data.getMutator().setType(idx(), MinorType.MAP); + getMapWriter(true).start(); + } + + @Override + public void end() { + getMapWriter(false).end(); + } + + @Override + public void startList() { + getListWriter(true).startList(); + data.getMutator().setType(idx(), MinorType.LIST); + } + + @Override + public void endList() { + getListWriter(true).endList(); + } + + private MapWriter getMapWriter(boolean create) { + if (create && mapWriter == null) { + mapWriter = new SingleMapWriter(data.getMap(), null, true, false); + mapWriter.setPosition(idx()); + writers.add(mapWriter); + } + return mapWriter; + } + + public MapWriter asMap() { + data.getMutator().setType(idx(), MinorType.MAP); + return getMapWriter(true); + } + + private ListWriter getListWriter(boolean create) { + if (create && listWriter == null) { + listWriter = new UnionListWriter(data.getList()); + listWriter.setPosition(idx()); + writers.add(listWriter); + } + return listWriter; + } + + public ListWriter asList() { + data.getMutator().setType(idx(), MinorType.LIST); + return getListWriter(true); + } + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + + <#if !minor.class?starts_with("Decimal")> + + private ${name}Writer ${name?uncap_first}Writer; + + private ${name}Writer get${name}Writer(boolean create) { + if (create && ${uncappedName}Writer == null) { + ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null); + ${uncappedName}Writer.setPosition(idx()); + writers.add(${uncappedName}Writer); + } + return ${uncappedName}Writer; + } + + public ${name}Writer as${name}() { + data.getMutator().setType(idx(), MinorType.${name?upper_case}); + return get${name}Writer(true); + } + + @Override + public void write(${name}Holder holder) { + data.getMutator().setType(idx(), MinorType.${name?upper_case}); + get${name}Writer(true).setPosition(idx()); + get${name}Writer(true).write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); + } + + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { + data.getMutator().setType(idx(), MinorType.${name?upper_case}); + get${name}Writer(true).setPosition(idx()); + get${name}Writer(true).write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); + } + + + + + public void writeNull() { + } + + @Override + public MapWriter map() { + data.getMutator().setType(idx(), MinorType.LIST); + getListWriter(true).setPosition(idx()); + return getListWriter(true).map(); + } + + @Override + public ListWriter list() { + data.getMutator().setType(idx(), MinorType.LIST); + getListWriter(true).setPosition(idx()); + return getListWriter(true).list(); + } + + @Override + public ListWriter list(String name) { + data.getMutator().setType(idx(), MinorType.MAP); + getMapWriter(true).setPosition(idx()); + return getMapWriter(true).list(name); + } + + @Override + public MapWriter map(String name) { + data.getMutator().setType(idx(), MinorType.MAP); + getMapWriter(true).setPosition(idx()); + return getMapWriter(true).map(name); + } + + <#list vv.types as type><#list type.minor as minor> + <#assign lowerName = minor.class?uncap_first /> + <#if lowerName == "int" ><#assign lowerName = "integer" /> + <#assign upperName = minor.class?upper_case /> + <#assign capName = minor.class?cap_first /> + <#if !minor.class?starts_with("Decimal")> + @Override + public ${capName}Writer ${lowerName}(String name) { + data.getMutator().setType(idx(), MinorType.MAP); + getMapWriter(true).setPosition(idx()); + return getMapWriter(true).${lowerName}(name); + } + + @Override + public ${capName}Writer ${lowerName}() { + data.getMutator().setType(idx(), MinorType.LIST); + getListWriter(true).setPosition(idx()); + return getListWriter(true).${lowerName}(); + } + + + + @Override + public void allocate() { + data.allocateNew(); + } + + @Override + public void clear() { + data.clear(); + } + + @Override + public void close() throws Exception { + data.close(); + } + + @Override + public MaterializedField getField() { + return data.getField(); + } + + @Override + public int getValueCapacity() { + return data.getValueCapacity(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index db7f68dc1b8..f85a776b907 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -160,6 +160,8 @@ public interface ExecConstants { public static String MONGO_READER_READ_NUMBERS_AS_DOUBLE = "store.mongo.read_numbers_as_double"; public static OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE, false); + public static BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", false); + // TODO: We need to add a feature that enables storage plugins to add their own options. Currently we have to declare // in core which is not right. Move this option and above two mongo plugin related options once we have the feature. public static String HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS = "store.hive.optimize_scan_with_native_readers"; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index df315b2e07c..bc6d807a611 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -87,12 +87,12 @@ private ExpressionTreeMaterializer() { }; public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext) { - return ExpressionTreeMaterializer.materialize(expr, batch, errorCollector, functionLookupContext, false); + return ExpressionTreeMaterializer.materialize(expr, batch, errorCollector, functionLookupContext, false, false); } public static LogicalExpression materializeAndCheckErrors(LogicalExpression expr, VectorAccessible batch, FunctionLookupContext functionLookupContext) throws SchemaChangeException { ErrorCollector collector = new ErrorCollectorImpl(); - LogicalExpression e = ExpressionTreeMaterializer.materialize(expr, batch, collector, functionLookupContext, false); + LogicalExpression e = ExpressionTreeMaterializer.materialize(expr, batch, collector, functionLookupContext, false, false); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); } @@ -100,8 +100,8 @@ public static LogicalExpression materializeAndCheckErrors(LogicalExpression expr } public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext, - boolean allowComplexWriterExpr) { - LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr), functionLookupContext); + boolean allowComplexWriterExpr, boolean unionTypeEnabled) { + LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr, unionTypeEnabled), functionLookupContext); if (!errorCollector.hasErrors()) { out = out.accept(ConditionalExprOptimizer.INSTANCE, null); @@ -190,11 +190,13 @@ private static class MaterializeVisitor extends AbstractExprVisitor g, JBlock sub, String body, H ValueReference parameter = parameters[i]; HoldingContainer inputVariable = inputVariables[i]; - if (parameter.isFieldReader && ! inputVariable.isReader() && ! Types.isComplex(inputVariable.getMajorType())) { + if (parameter.isFieldReader && ! inputVariable.isReader() && ! Types.isComplex(inputVariable.getMajorType()) && inputVariable.getMinorType() != MinorType.UNION) { JType singularReaderClass = g.getModel()._ref(TypeHelper.getHolderReaderImpl(inputVariable.getMajorType().getMinorType(), inputVariable.getMajorType().getMode())); JType fieldReadClass = g.getModel()._ref(FieldReader.class); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MappifyUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MappifyUtility.java index e27234fdb22..b7877df1693 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MappifyUtility.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MappifyUtility.java @@ -46,7 +46,7 @@ public static DrillBuf mappify(FieldReader reader, BaseWriter.ComplexWriter writ throw new DrillRuntimeException("kvgen function only supports Simple maps as input"); } BaseWriter.ListWriter listWriter = writer.rootAsList(); - listWriter.start(); + listWriter.startList(); BaseWriter.MapWriter mapWriter = listWriter.map(); // Iterate over the fields in the map @@ -79,7 +79,7 @@ public static DrillBuf mappify(FieldReader reader, BaseWriter.ComplexWriter writ mapWriter.end(); } - listWriter.end(); + listWriter.endList(); return buffer; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java new file mode 100644 index 00000000000..23a5004e9a3 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl; + +import io.netty.buffer.DrillBuf; +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.UnionHolder; +import org.apache.drill.exec.expr.holders.IntHolder; +import org.apache.drill.exec.expr.holders.VarCharHolder; +import org.apache.drill.exec.vector.complex.impl.UnionReader; +import org.apache.drill.exec.vector.complex.reader.FieldReader; + +import javax.inject.Inject; + +public class UnionFunctions { + + @FunctionTemplate(names = {"typeString"}, + scope = FunctionTemplate.FunctionScope.SIMPLE, + nulls = NullHandling.NULL_IF_NULL) + public static class FromType implements DrillSimpleFunc { + + @Param + IntHolder in; + @Output + VarCharHolder out; + @Inject + DrillBuf buffer; + + public void setup() {} + + public void eval() { + + VarCharHolder h = org.apache.drill.exec.vector.ValueHolderHelper.getVarCharHolder(buffer, org.apache.drill.common.types.MinorType.valueOf(in.value).toString()); + out.buffer = h.buffer; + out.start = h.start; + out.end = h.end; + } + } + + @FunctionTemplate(names = {"type"}, + scope = FunctionTemplate.FunctionScope.SIMPLE, + nulls = NullHandling.NULL_IF_NULL) + public static class ToType implements DrillSimpleFunc { + + @Param + VarCharHolder input; + @Output + IntHolder out; + + public void setup() {} + + public void eval() { + + out.value = input.getType().getMinorType().getNumber(); + byte[] b = new byte[input.end - input.start]; + input.buffer.getBytes(input.start, b, 0, b.length); + String type = new String(b); + out.value = org.apache.drill.common.types.MinorType.valueOf(type.toUpperCase()).getNumber(); + } + } + + @FunctionTemplate(names = {"typeOf"}, + scope = FunctionTemplate.FunctionScope.SIMPLE, + nulls = NullHandling.INTERNAL) + public static class GetType implements DrillSimpleFunc { + + @Param + FieldReader input; + @Output + IntHolder out; + + public void setup() {} + + public void eval() { + + out.value = input.isSet() ? input.getType().getMinorType().getNumber() : 0; + + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class CastUnionToUnion implements DrillSimpleFunc{ + + @Param FieldReader in; + @Output + UnionHolder out; + + public void setup() {} + + public void eval() { + out.reader = in; + out.isSet = in.isSet() ? 1 : 0; + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java new file mode 100644 index 00000000000..84cdefb7bb0 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.holders; + +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.vector.complex.reader.FieldReader; + +public class UnionHolder implements ValueHolder { + public static final MajorType TYPE = Types.optional(MinorType.UNION); + public FieldReader reader; + public int isSet; + + public MajorType getType() { + return reader.getType(); + } + + public boolean isSet() { + return isSet == 1; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java index 432e06b8e8d..fd16bc84153 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java @@ -23,6 +23,7 @@ import org.apache.drill.common.expression.ErrorCollector; import org.apache.drill.common.expression.ErrorCollectorImpl; import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.ClassTransformationException; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.ClassGenerator; @@ -176,7 +177,8 @@ protected Filterer generateSV2Filterer() throws SchemaChangeException { final List transfers = Lists.newArrayList(); final ClassGenerator cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION2, context.getFunctionRegistry()); - final LogicalExpression expr = ExpressionTreeMaterializer.materialize(popConfig.getExpr(), incoming, collector, context.getFunctionRegistry()); + final LogicalExpression expr = ExpressionTreeMaterializer.materialize(popConfig.getExpr(), incoming, collector, + context.getFunctionRegistry(), false, unionTypeEnabled); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index 3a8b73580cb..9f64626e30d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -288,7 +288,7 @@ protected boolean setupNewSchema() throws SchemaChangeException { final IntOpenHashSet transferFieldIds = new IntOpenHashSet(); final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn())); - final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression)ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true); + final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression)ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true, false); final TransferPair tp = getFlattenFieldTransferPair(flattenExpr.getRef()); if (tp != null) { @@ -315,7 +315,7 @@ protected boolean setupNewSchema() throws SchemaChangeException { } } - final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true); + final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true, false); final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType()); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index 5b5c90d47f2..dfca892ff06 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -37,6 +37,7 @@ import org.apache.drill.common.logical.data.NamedExpression; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.ClassTransformationException; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.ClassGenerator; @@ -63,7 +64,6 @@ import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.FixedWidthVector; import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.exec.vector.complex.AbstractContainerVector; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; import com.carrotsearch.hppc.IntOpenHashSet; @@ -381,7 +381,8 @@ protected boolean setupNewSchema() throws SchemaChangeException { } } - final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true); + final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, + collector, context.getFunctionRegistry(), true, unionTypeEnabled); final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType()); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java index d8f703ed485..aaa6f9e5b77 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java @@ -22,6 +22,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.memory.OutOfMemoryException; import org.apache.drill.exec.ops.FragmentContext; @@ -30,6 +31,7 @@ import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.record.selection.SelectionVector2; import org.apache.drill.exec.record.selection.SelectionVector4; +import org.apache.drill.exec.server.options.OptionValue; public abstract class AbstractRecordBatch implements CloseableRecordBatch { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(new Object() {}.getClass().getEnclosingClass()); @@ -39,6 +41,7 @@ public abstract class AbstractRecordBatch implements protected final FragmentContext context; protected final OperatorContext oContext; protected final OperatorStats stats; + protected final boolean unionTypeEnabled; protected BatchState state; @@ -62,6 +65,12 @@ protected AbstractRecordBatch(final T popConfig, final FragmentContext context, } else { state = BatchState.FIRST; } + OptionValue option = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE.getOptionName()); + if (option != null) { + unionTypeEnabled = option.bool_val; + } else { + unionTypeEnabled = false; + } } protected static enum BatchState { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java index 6b1d178796e..21b3adc9a91 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java @@ -19,11 +19,15 @@ import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.MinorType; +import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.complex.AbstractContainerVector; import org.apache.drill.exec.vector.complex.AbstractMapVector; import org.apache.drill.exec.vector.complex.MapVector; +import org.apache.drill.exec.vector.complex.impl.UnionVector; public class SimpleVectorWrapper implements VectorWrapper{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleVectorWrapper.class); @@ -103,6 +107,13 @@ public TypedFieldId getFieldIdIfMatches(int id, SchemaPath expectedPath) { } PathSegment seg = expectedPath.getRootSegment(); + if (v instanceof UnionVector) { + TypedFieldId.Builder builder = TypedFieldId.newBuilder(); + builder.addId(id).remainder(expectedPath.getRootSegment().getChild()); + builder.finalType(Types.optional(TypeProtos.MinorType.UNION)); + builder.intermediateType(Types.optional(TypeProtos.MinorType.UNION)); + return builder.build(); + } else if (v instanceof AbstractContainerVector) { // we're looking for a multi path. AbstractContainerVector c = (AbstractContainerVector) v; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java index 29dd6a53056..8c602b33627 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java @@ -77,6 +77,7 @@ public class ResolverTypePrecedence { precedenceMap.put(MinorType.INTERVALDAY, i+= 2); precedenceMap.put(MinorType.INTERVALYEAR, i+= 2); precedenceMap.put(MinorType.INTERVAL, i+= 2); + precedenceMap.put(MinorType.UNION, i += 2); MAX_IMPLICIT_CAST_COST = i; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java index 3278d3e6419..7ee8ebed33a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java @@ -24,6 +24,8 @@ import java.util.Map; import java.util.Set; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; @@ -750,6 +752,8 @@ private static void initTypeRules() { rule.add(MinorType.VARBINARY); rule.add(MinorType.FIXEDBINARY); rules.put(MinorType.VARBINARY, rule); + + rules.put(MinorType.UNION, Sets.newHashSet(MinorType.UNION)); } public static boolean isCastableWithNullHandling(MajorType from, MajorType to, NullHandling nullHandling) { @@ -792,10 +796,16 @@ public static DataMode getLeastRestrictiveDataMode(List dataModes) { public static MinorType getLeastRestrictiveType(List types) { assert types.size() >= 2; MinorType result = types.get(0); + if (result == MinorType.UNION) { + return result; + } int resultPrec = ResolverTypePrecedence.precedenceMap.get(result); for (int i = 1; i < types.size(); i++) { MinorType next = types.get(i); + if (next == MinorType.UNION) { + return next; + } if (next == result) { // both args are of the same type; continue continue; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index b3bab2a672d..5e16483bb6f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -92,6 +92,7 @@ public class SystemOptionManager extends BaseOptionManager { ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR, ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR, ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR, + ExecConstants.ENABLE_UNION_TYPE, ExecConstants.TEXT_ESTIMATED_ROW_SIZE, ExecConstants.JSON_EXTENDED_TYPES, ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR, diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/MapOrListWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/MapOrListWriter.java index 1a94452a1d2..e74021b94ea 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/MapOrListWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/MapOrListWriter.java @@ -49,7 +49,7 @@ void start() { if (map != null) { map.start(); } else { - list.start(); + list.startList(); } } @@ -57,7 +57,7 @@ void end() { if (map != null) { map.end(); } else { - list.end(); + list.endList(); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java index 4d51199eb9d..0e3c9080021 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java @@ -59,6 +59,7 @@ public class JSONRecordReader extends AbstractRecordReader { private final FragmentContext fragmentContext; private final boolean enableAllTextMode; private final boolean readNumbersAsDouble; + private final boolean unionEnabled; /** * Create a JSON Record Reader that uses a file based input stream. @@ -108,6 +109,7 @@ private JSONRecordReader(final FragmentContext fragmentContext, final String inp // only enable all text mode if we aren't using embedded content mode. this.enableAllTextMode = embeddedContent == null && fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR); this.readNumbersAsDouble = fragmentContext.getOptions().getOption(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE).bool_val; + this.unionEnabled = fragmentContext.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE); setColumns(columns); } @@ -118,7 +120,7 @@ public void setup(final OperatorContext context, final OutputMutator output) thr this.stream = fileSystem.openPossiblyCompressedStream(hadoopPath); } - this.writer = new VectorContainerWriter(output); + this.writer = new VectorContainerWriter(output, unionEnabled); if (isSkipQuery()) { this.jsonReader = new CountingJsonReader(fragmentContext.getManagedBuffer()); } else { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java index ea45653d222..9f7d2b30683 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.store.RecordWriter; import org.apache.drill.exec.vector.complex.fn.BasicJsonOutput; import org.apache.drill.exec.vector.complex.fn.ExtendedJsonOutput; +import org.apache.drill.exec.vector.complex.fn.JsonWriter; import org.apache.drill.exec.vector.complex.reader.FieldReader; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -128,6 +129,29 @@ public void writeField() throws IOException { } } + @Override + public FieldConverter getNewUnionConverter(int fieldId, String fieldName, FieldReader reader) { + return new UnionJsonConverter(fieldId, fieldName, reader); + } + + public class UnionJsonConverter extends FieldConverter { + + public UnionJsonConverter(int fieldId, String fieldName, FieldReader reader) { + super(fieldId, fieldName, reader); + } + + @Override + public void startField() throws IOException { + gen.writeFieldName(fieldName); + } + + @Override + public void writeField() throws IOException { + JsonWriter writer = new JsonWriter(gen); + writer.write(reader); + } + } + @Override public FieldConverter getNewRepeatedMapConverter(int fieldId, String fieldName, FieldReader reader) { return new RepeatedMapJsonConverter(fieldId, fieldName, reader); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java new file mode 100644 index 00000000000..ecf2596edba --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.accessor; + +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.vector.complex.impl.UnionVector; +import org.apache.drill.exec.vector.complex.impl.UnionWriter; +import org.apache.drill.exec.vector.complex.reader.FieldReader; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; + +public class UnionSqlAccessor extends AbstractSqlAccessor { + + FieldReader reader; + + public UnionSqlAccessor(UnionVector vector) { + reader = vector.getReader(); + } + + @Override + public boolean isNull(int rowOffset) { + reader.setPosition(rowOffset); + return reader.isSet(); + } + + @Override + public BigDecimal getBigDecimal(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readBigDecimal(); + } + + @Override + public boolean getBoolean(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readBoolean(); + } + + @Override + public byte getByte(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readByte(); + } + + @Override + public byte[] getBytes(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readByteArray(); + } + + @Override + public double getDouble(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readDouble(); + } + + @Override + public float getFloat(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readFloat(); + } + + @Override + public int getInt(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readInteger(); + } + + @Override + public long getLong(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readLong(); + } + + @Override + public short getShort(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readShort(); + } + + @Override + public char getChar(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return reader.readCharacter(); + } + + @Override + public String getString(int rowOffset) throws InvalidAccessException{ + reader.setPosition(rowOffset); + return getObject(rowOffset).toString(); + } + + @Override + public Object getObject(int rowOffset) throws InvalidAccessException { + reader.setPosition(rowOffset); + return reader.readObject(); + } + + @Override + public MajorType getType() { + return Types.optional(MinorType.UNION); + } + + @Override + public Class getObjectClass() { + return Object.class; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java new file mode 100644 index 00000000000..ccd6239ec7b --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java @@ -0,0 +1,292 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.vector.complex; + +import com.google.common.collect.ObjectArrays; +import io.netty.buffer.DrillBuf; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; +import org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.TransferPair; +import org.apache.drill.exec.util.CallBack; +import org.apache.drill.exec.util.JsonStringArrayList; +import org.apache.drill.exec.vector.BaseValueVector; +import org.apache.drill.exec.vector.UInt1Vector; +import org.apache.drill.exec.vector.UInt4Vector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VectorDescriptor; +import org.apache.drill.exec.vector.complex.impl.ComplexCopier; +import org.apache.drill.exec.vector.complex.impl.UnionListReader; +import org.apache.drill.exec.vector.complex.impl.UnionListWriter; +import org.apache.drill.exec.vector.complex.impl.UnionVector; +import org.apache.drill.exec.vector.complex.reader.FieldReader; +import org.apache.drill.exec.vector.complex.writer.FieldWriter; + +import java.util.List; + +public class ListVector extends BaseRepeatedValueVector { + + UInt4Vector offsets; + protected final UInt1Vector bits; + Mutator mutator = new Mutator(); + Accessor accessor = new Accessor(); + UnionListWriter writer; + UnionListReader reader; + + public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { + super(field, allocator, new UnionVector(field, allocator, callBack)); + this.bits = new UInt1Vector(MaterializedField.create("$bits$", Types.required(MinorType.UINT1)), allocator); + offsets = getOffsetVector(); + this.field.addChild(getDataVector().getField()); + this.writer = new UnionListWriter(this); + this.reader = new UnionListReader(this); + } + + public UnionListWriter getWriter() { + return writer; + } + + @Override + public void allocateNew() throws OutOfMemoryRuntimeException { + super.allocateNewSafe(); + } + + public void transferTo(ListVector target) { + offsets.makeTransferPair(target.offsets).transfer(); + bits.makeTransferPair(target.bits).transfer(); + getDataVector().makeTransferPair(target.getDataVector()).transfer(); + } + + public void copyFrom(int inIndex, int outIndex, ListVector from) { + FieldReader in = from.getReader(); + in.setPosition(inIndex); + FieldWriter out = getWriter(); + out.setPosition(outIndex); + ComplexCopier copier = new ComplexCopier(in, out); + copier.write(); + } + + @Override + public UnionVector getDataVector() { + return (UnionVector) vector; + } + + @Override + public TransferPair getTransferPair(FieldReference ref) { + return new TransferImpl(field.withPath(ref)); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return new TransferImpl((ListVector) target); + } + + private class TransferImpl implements TransferPair { + + ListVector to; + + public TransferImpl(MaterializedField field) { + to = new ListVector(field, allocator, null); + } + + public TransferImpl(ListVector to) { + this.to = to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + + } + + @Override + public ValueVector getTo() { + return to; + } + + @Override + public void copyValueSafe(int from, int to) { + this.to.copyFrom(from, to, ListVector.this); + } + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public boolean allocateNewSafe() { + /* boolean to keep track if all the memory allocation were successful + * Used in the case of composite vectors when we need to allocate multiple + * buffers for multiple vectors. If one of the allocations failed we need to + * clear all the memory that we allocated + */ + boolean success = false; + try { + if (!offsets.allocateNewSafe()) { + return false; + } + success = vector.allocateNewSafe(); + success = success && bits.allocateNewSafe(); + } finally { + if (!success) { + clear(); + } + } + offsets.zeroVector(); + bits.zeroVector(); + return success; + } + + @Override + protected UserBitShared.SerializedField.Builder getMetadataBuilder() { + return getField().getAsBuilder() + .setValueCount(getAccessor().getValueCount()) + .setBufferLength(getBufferSize()) + .addChild(offsets.getMetadata()) + .addChild(bits.getMetadata()) + .addChild(vector.getMetadata()); + } + + @Override + public int getBufferSize() { + if (getAccessor().getValueCount() == 0) { + return 0; + } + return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize(); + } + + @Override + public void clear() { + offsets.clear(); + vector.clear(); + bits.clear(); + lastSet = 0; + super.clear(); + } + + @Override + public DrillBuf[] getBuffers(boolean clear) { + final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false), + vector.getBuffers(false), DrillBuf.class), DrillBuf.class); + if (clear) { + for (DrillBuf buffer:buffers) { + buffer.retain(); + } + clear(); + } + return buffers; + } + + @Override + public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { + final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0); + offsets.load(offsetMetadata, buffer); + + final int offsetLength = offsetMetadata.getBufferLength(); + final UserBitShared.SerializedField bitMetadata = metadata.getChild(1); + final int bitLength = bitMetadata.getBufferLength(); + bits.load(bitMetadata, buffer.slice(offsetLength, bitLength)); + + final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2); + if (getDataVector() == DEFAULT_DATA_VECTOR) { + addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType())); + } + + final int vectorLength = vectorMetadata.getBufferLength(); + vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength)); + } + + private int lastSet; + + public class Accessor extends BaseRepeatedAccessor { + + @Override + public Object getObject(int index) { + if (bits.getAccessor().isNull(index)) { + return null; + } + final List vals = new JsonStringArrayList<>(); + final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); + final int start = offsetsAccessor.get(index); + final int end = offsetsAccessor.get(index + 1); + final UnionVector.Accessor valuesAccessor = getDataVector().getAccessor(); + for(int i = start; i < end; i++) { + vals.add(valuesAccessor.getObject(i)); + } + return vals; + } + + @Override + public boolean isNull(int index) { + return bits.getAccessor().get(index) == 0; + } + } + + public class Mutator extends BaseRepeatedMutator { + public void setNotNull(int index) { + bits.getMutator().setSafe(index, 1); + lastSet = index + 1; + } + + @Override + public void startNewValue(int index) { + for (int i = lastSet; i <= index; i++) { + offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); + } + setNotNull(index); + lastSet = index + 1; + } + + @Override + public void setValueCount(int valueCount) { + // TODO: populate offset end points + if (valueCount == 0) { + offsets.getMutator().setValueCount(0); + } else { + for (int i = lastSet; i < valueCount; i++) { + offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); + } + offsets.getMutator().setValueCount(valueCount + 1); + } + final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); + vector.getMutator().setValueCount(childValueCount); + bits.getMutator().setValueCount(valueCount); + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java index 603776daba9..d55b1d39baa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java @@ -275,83 +275,86 @@ private void consumeEntireNextValue() throws IOException { private void writeData(MapWriter map, FieldSelection selection, boolean moveForward) throws IOException { // map.start(); - outside: while (true) { - - JsonToken t; - if(moveForward){ - t = parser.nextToken(); - }else{ - t = parser.getCurrentToken(); - moveForward = true; - } + try { + outside: + while (true) { + + JsonToken t; + if (moveForward) { + t = parser.nextToken(); + } else { + t = parser.getCurrentToken(); + moveForward = true; + } - if (t == JsonToken.NOT_AVAILABLE || t == JsonToken.END_OBJECT) { - return; - } + if (t == JsonToken.NOT_AVAILABLE || t == JsonToken.END_OBJECT) { + return; + } - assert t == JsonToken.FIELD_NAME : String.format("Expected FIELD_NAME but got %s.", t.name()); + assert t == JsonToken.FIELD_NAME : String.format("Expected FIELD_NAME but got %s.", t.name()); - final String fieldName = parser.getText(); - this.currentFieldName = fieldName; - FieldSelection childSelection = selection.getChild(fieldName); - if (childSelection.isNeverValid()) { - consumeEntireNextValue(); - continue outside; - } - - switch (parser.nextToken()) { - case START_ARRAY: - writeData(map.list(fieldName)); - break; - case START_OBJECT: - if (!writeMapDataIfTyped(map, fieldName)) { - writeData(map.map(fieldName), childSelection, false); + final String fieldName = parser.getText(); + this.currentFieldName = fieldName; + FieldSelection childSelection = selection.getChild(fieldName); + if (childSelection.isNeverValid()) { + consumeEntireNextValue(); + continue outside; } - break; - case END_OBJECT: - break outside; - case VALUE_FALSE: { - map.bit(fieldName).writeBit(0); - atLeastOneWrite = true; - break; - } - case VALUE_TRUE: { - map.bit(fieldName).writeBit(1); - atLeastOneWrite = true; - break; - } - case VALUE_NULL: - // do nothing as we don't have a type. - break; - case VALUE_NUMBER_FLOAT: - map.float8(fieldName).writeFloat8(parser.getDoubleValue()); - atLeastOneWrite = true; - break; - case VALUE_NUMBER_INT: - if (this.readNumbersAsDouble) { - map.float8(fieldName).writeFloat8(parser.getDoubleValue()); + switch (parser.nextToken()) { + case START_ARRAY: + writeData(map.list(fieldName)); + break; + case START_OBJECT: + if (!writeMapDataIfTyped(map, fieldName)) { + writeData(map.map(fieldName), childSelection, false); + } + break; + case END_OBJECT: + break outside; + + case VALUE_FALSE: { + map.bit(fieldName).writeBit(0); + atLeastOneWrite = true; + break; } - else { - map.bigInt(fieldName).writeBigInt(parser.getLongValue()); + case VALUE_TRUE: { + map.bit(fieldName).writeBit(1); + atLeastOneWrite = true; + break; + } + case VALUE_NULL: + // do nothing as we don't have a type. + break; + case VALUE_NUMBER_FLOAT: + map.float8(fieldName).writeFloat8(parser.getDoubleValue()); + atLeastOneWrite = true; + break; + case VALUE_NUMBER_INT: + if (this.readNumbersAsDouble) { + map.float8(fieldName).writeFloat8(parser.getDoubleValue()); + } else { + map.bigInt(fieldName).writeBigInt(parser.getLongValue()); + } + atLeastOneWrite = true; + break; + case VALUE_STRING: + handleString(parser, map, fieldName); + atLeastOneWrite = true; + break; + + default: + throw + getExceptionWithContext( + UserException.dataReadError(), currentFieldName, null) + .message("Unexpected token %s", parser.getCurrentToken()) + .build(logger); } - atLeastOneWrite = true; - break; - case VALUE_STRING: - handleString(parser, map, fieldName); - atLeastOneWrite = true; - break; - default: - throw - getExceptionWithContext( - UserException.dataReadError(), currentFieldName, null) - .message("Unexpected token %s", parser.getCurrentToken()) - .build(logger); } - + } finally { + map.end(); } - map.end(); } @@ -463,8 +466,8 @@ private void handleString(JsonParser parser, ListWriter writer) throws IOExcepti writer.varChar().writeVarChar(0, workingBuffer.prepareVarCharHolder(parser.getText()), workingBuffer.getBuf()); } - private void writeData(ListWriter list) { - list.start(); + private void writeData(ListWriter list) throws IOException { + list.startList(); outside: while (true) { try { switch (parser.nextToken()) { @@ -523,12 +526,12 @@ private void writeData(ListWriter list) { throw getExceptionWithContext(e, this.currentFieldName, null).build(logger); } } - list.end(); + list.endList(); } private void writeDataAllText(ListWriter list) throws IOException { - list.start(); + list.startList(); outside: while (true) { switch (parser.nextToken()) { @@ -562,7 +565,7 @@ private void writeDataAllText(ListWriter list) throws IOException { .build(logger); } } - list.end(); + list.endList(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java index 8309bf34702..6ff7d905f90 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java @@ -47,6 +47,10 @@ public JsonWriter(OutputStream out, boolean pretty, boolean useExtendedOutput) t } + public JsonWriter(JsonOutput gen) { + this.gen = gen; + } + public void write(FieldReader reader) throws JsonGenerationException, IOException{ writeValue(reader); gen.flush(); @@ -109,7 +113,11 @@ private void writeValue(FieldReader reader) throws JsonGenerationException, IOEx case LIST: // this is a pseudo class, doesn't actually contain the real reader so we have to drop down. - writeValue(reader.reader()); + gen.writeStartArray(); + while (reader.next()) { + writeValue(reader.reader()); + } + gen.writeEndArray(); break; case MAP: gen.writeStartObject(); @@ -125,6 +133,7 @@ private void writeValue(FieldReader reader) throws JsonGenerationException, IOEx gen.writeEndObject(); break; case NULL: + case LATE: gen.writeUntypedNull(); break; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java index 22addc97e2d..fea326c4c96 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java @@ -21,6 +21,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.Types; +import org.apache.drill.exec.expr.holders.UnionHolder; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.vector.complex.reader.FieldReader; @@ -71,4 +72,20 @@ public boolean next() { public int size() { throw new IllegalStateException("The current reader doesn't support getting size information."); } + + @Override + public void read(UnionHolder holder) { + holder.reader = this; + holder.isSet = this.isSet() ? 1 : 0; + } + + @Override + public void read(int index, UnionHolder holder) { + throw new IllegalStateException("The current reader doesn't support reading union type"); + } + + @Override + public void copyAsValue(UnionWriter writer) { + throw new IllegalStateException("The current reader doesn't support reading union type"); + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java index 88a56f8707e..23bcfcb635d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java @@ -38,13 +38,19 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri Mode mode = Mode.INIT; private final String name; + private final boolean unionEnabled; private enum Mode { INIT, MAP, LIST }; - public ComplexWriterImpl(String name, MapVector container){ + public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){ super(null); this.name = name; this.container = container; + this.unionEnabled = unionEnabled; + } + + public ComplexWriterImpl(String name, MapVector container){ + this(name, container, false); } @Override @@ -120,7 +126,7 @@ public MapWriter directMap(){ case INIT: MapVector map = (MapVector) container; - mapRoot = new SingleMapWriter(map, this); + mapRoot = new SingleMapWriter(map, this, unionEnabled, false); mapRoot.setPosition(idx()); mode = Mode.MAP; break; @@ -141,7 +147,7 @@ public MapWriter rootAsMap() { case INIT: MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class); - mapRoot = new SingleMapWriter(map, this); + mapRoot = new SingleMapWriter(map, this, unionEnabled, false); mapRoot.setPosition(idx()); mode = Mode.MAP; break; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java new file mode 100644 index 00000000000..fef33259bd3 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java @@ -0,0 +1,90 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.vector.complex.impl; + +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.exec.expr.holders.UnionHolder; +import org.apache.drill.exec.vector.UInt4Vector; +import org.apache.drill.exec.vector.complex.ListVector; +import org.apache.drill.exec.vector.complex.reader.FieldReader; + +public class UnionListReader extends AbstractFieldReader { + + private ListVector vector; + private UnionVector data; + private UInt4Vector offsets; + private UnionReader reader; + + public UnionListReader(ListVector vector) { + this.vector = vector; + this.data = vector.getDataVector(); + this.offsets = vector.getOffsetVector(); + this.reader = (UnionReader) data.getReader(); + } + + @Override + public boolean isSet() { + return true; + } + + @Override + public MajorType getType() { + return reader.getType(); + } + + private int currentOffset; + private int maxOffset; + + @Override + public void setPosition(int index) { + super.setPosition(index); + currentOffset = offsets.getAccessor().get(index) - 1; + maxOffset = offsets.getAccessor().get(index + 1); + } + + @Override + public FieldReader reader() { + return reader; + } + + @Override + public Object readObject() { + return vector.getAccessor().getObject(idx()); + } + + @Override + public void read(int index, UnionHolder holder) { + setPosition(idx()); + for (int i = -1; i < index; i++) { + next(); + } + holder.reader = reader; + holder.isSet = reader.isSet() ? 1 : 0; + } + + @Override + public boolean next() { + if (currentOffset + 1 < maxOffset) { + reader.setPosition(++currentOffset); + return true; + } else { + return false; + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java index 95c651f5186..6bc2e05a571 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java @@ -33,11 +33,15 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple private final SpecialMapVector mapVector; private final OutputMutator mutator; - public VectorContainerWriter(OutputMutator mutator) { + public VectorContainerWriter(OutputMutator mutator, boolean unionEnabled) { super(null); this.mutator = mutator; mapVector = new SpecialMapVector(mutator.getCallBack()); - mapRoot = new SingleMapWriter(mapVector, this); + mapRoot = new SingleMapWriter(mapVector, this, unionEnabled, false); + } + + public VectorContainerWriter(OutputMutator mutator) { + this(mutator, false); } @Override diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java index e3591b660bf..51b909b366e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java @@ -98,4 +98,8 @@ public CallBack getCallBack() { return null; } + public VectorContainer getContainer() { + return container; + } + } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index bd9cea16ad7..0ae8945c8aa 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -104,6 +104,7 @@ public void testSplitAndTransferFailure() throws Exception { .baselineValues(listOf(testVal)) .go(); + test("select flatten(config) as flat from cp.`/store/json/null_list_v2.json`"); testBuilder() .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list_v2.json`") .ordered() @@ -377,5 +378,108 @@ private void testExistentColumns(RecordBatchLoader batchLoader) throws SchemaCha assertEquals("[4,5,6]", vw.getValueVector().getAccessor().getObject(2).toString()); } + @Test + public void testSelectStarWithUnionType() throws Exception { + try { + String query = "select * from cp.`jsoninput/union/a.json`"; + testBuilder() + .sqlQuery(query) + .ordered() + .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") + .baselineColumns("field1", "field2") + .baselineValues( + 1L, 1.2 + ) + .baselineValues( + listOf(2L), 1.2 + ) + .baselineValues( + mapOf("inner1", 3L, "inner2", 4L), listOf(3L, 4.0, "5") + ) + .baselineValues( + mapOf("inner1", 3L, + "inner2", listOf( + mapOf( + "innerInner1", 1L, + "innerInner2", + listOf( + 3L, + "a" + ) + ) + ) + ), + listOf( + mapOf("inner3", 7L), + 4.0, + "5", + mapOf("inner4", 9L), + listOf( + mapOf( + "inner5", 10L, + "inner6", 11L + ), + mapOf( + "inner5", 12L, + "inner7", 13L + ) + ) + ) + ).go(); + } finally { + testNoResult("alter session set `exec.enable_union_type` = false"); + } + } + + @Test + public void testSelectFromListWithCase() throws Exception { + String query = "select a from (select case when typeOf(field2) = type('list') then asBigInt(field2[4][1].inner7) end a from cp.`jsoninput/union/a.json`) where a is not null"; + try { + testBuilder() + .sqlQuery(query) + .ordered() + .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") + .baselineColumns("a") + .baselineValues(13L) + .go(); + } finally { + testNoResult("alter session set `exec.enable_union_type` = false"); + } + } + + @Test + public void testTypeCase() throws Exception { + String query = "select case typeOf(field1) when type('bigint') then asBigInt(field1) when type('list') then asBigInt(field1[0]) when type('map') then asBigInt(t.field1.inner1) end f1 from cp.`jsoninput/union/a.json` t"; + try { + testBuilder() + .sqlQuery(query) + .ordered() + .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") + .baselineColumns("f1") + .baselineValues(1L) + .baselineValues(2L) + .baselineValues(3L) + .baselineValues(3L) + .go(); + } finally { + testNoResult("alter session set `exec.enable_union_type` = false"); + } + } + + @Test + public void testSumWithTypeCase() throws Exception { + String query = "select sum(f1) sum_f1 from (select case typeOf(field1) when type('bigint') then asBigInt(field1) when type('list') then asBigInt(field1[0]) when type('map') then asBigInt(t.field1.inner1) end f1 from cp.`jsoninput/union/a.json` t)"; + try { + testBuilder() + .sqlQuery(query) + .ordered() + .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") + .baselineColumns("sum_f1") + .baselineValues(9L) + .go(); + } finally { + testNoResult("alter session set `exec.enable_union_type` = false"); + } + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestRepeated.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestRepeated.java index df0ce142fd9..bd4731a61a4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestRepeated.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestRepeated.java @@ -139,12 +139,12 @@ public void listOfList() throws Exception { { final MapWriter map = writer.rootAsMap(); final ListWriter list = map.list("a"); - list.start(); + list.startList(); final ListWriter innerList = list.list(); final IntWriter innerInt = innerList.integer(); - innerList.start(); + innerList.startList(); final IntHolder holder = new IntHolder(); @@ -155,16 +155,16 @@ public void listOfList() throws Exception { holder.value = 3; innerInt.write(holder); - innerList.end(); - innerList.start(); + innerList.endList(); + innerList.startList(); holder.value = 4; innerInt.write(holder); holder.value = 5; innerInt.write(holder); - innerList.end(); - list.end(); + innerList.endList(); + list.endList(); final IntWriter numCol = map.integer("nums"); holder.value = 14; @@ -192,12 +192,12 @@ public void listOfList() throws Exception { final MapWriter map = writer.rootAsMap(); final ListWriter list = map.list("a"); - list.start(); + list.startList(); final ListWriter innerList = list.list(); final IntWriter innerInt = innerList.integer(); - innerList.start(); + innerList.startList(); final IntHolder holder = new IntHolder(); @@ -208,16 +208,16 @@ public void listOfList() throws Exception { holder.value = -3; innerInt.write(holder); - innerList.end(); - innerList.start(); + innerList.endList(); + innerList.startList(); holder.value = -4; innerInt.write(holder); holder.value = -5; innerInt.write(holder); - innerList.end(); - list.end(); + innerList.endList(); + list.endList(); final IntWriter numCol = map.integer("nums"); holder.value = -28; diff --git a/exec/java-exec/src/test/resources/jsoninput/union/a.json b/exec/java-exec/src/test/resources/jsoninput/union/a.json new file mode 100644 index 00000000000..438dba3c685 --- /dev/null +++ b/exec/java-exec/src/test/resources/jsoninput/union/a.json @@ -0,0 +1,52 @@ +{ + field1: 1, + field2: 1.2 +} +{ + field1: [ + 2 + ], + field2: 1.2 +} +{ + field1: { + inner1: 3, + inner2: 4 + }, + field2: [ + 3, + 4.0, + "5" + ] +} +{ + field1: { + inner1: 3, + inner2: [ + { + innerInner1: 1, + innerInner2: [ + 3, + "a" + ] + } + ] + }, + field2: [ + { + inner3: 7 + }, + 4.0, + "5", + { + inner4: 9 + }, + [{ + inner5: 10, + inner6: 11 + },{ + inner5: 12, + inner7: 13 + }] + ] +} \ No newline at end of file diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java index 35890ec2eb5..b5634bf6cd2 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java @@ -70,7 +70,6 @@ public void testSimpleQuerySingleFile() throws Exception{ + "from dfs_test.`%s/../../sample-data/regionsSF/`", WORKING_PATH)); } - @Test public void testSimpleQueryMultiFile() throws Exception{ testQuery(String.format("select R_REGIONKEY, R_NAME " diff --git a/protocol/src/main/java/org/apache/drill/common/types/MinorType.java b/protocol/src/main/java/org/apache/drill/common/types/MinorType.java index 423ed533329..16bd53e90df 100644 --- a/protocol/src/main/java/org/apache/drill/common/types/MinorType.java +++ b/protocol/src/main/java/org/apache/drill/common/types/MinorType.java @@ -58,7 +58,8 @@ public enum MinorType implements com.dyuproject.protostuff.EnumLite INTERVALYEAR(38), INTERVALDAY(39), LIST(40), - GENERIC_OBJECT(41); + GENERIC_OBJECT(41), + UNION(42); public final int number; @@ -113,6 +114,7 @@ public static MinorType valueOf(int number) case 39: return INTERVALDAY; case 40: return LIST; case 41: return GENERIC_OBJECT; + case 42: return UNION; default: return null; } } diff --git a/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java b/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java index 74ac444485d..be7fe7443d6 100644 --- a/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java +++ b/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java @@ -317,6 +317,10 @@ public enum MinorType * GENERIC_OBJECT = 41; */ GENERIC_OBJECT(36, 41), + /** + * UNION = 42; + */ + UNION(37, 42), ; /** @@ -606,6 +610,10 @@ public enum MinorType * GENERIC_OBJECT = 41; */ public static final int GENERIC_OBJECT_VALUE = 41; + /** + * UNION = 42; + */ + public static final int UNION_VALUE = 42; public final int getNumber() { return value; } @@ -649,6 +657,7 @@ public static MinorType valueOf(int value) { case 39: return INTERVALDAY; case 40: return LIST; case 41: return GENERIC_OBJECT; + case 42: return UNION; default: return null; } } @@ -1780,7 +1789,7 @@ public Builder clearTimeZone() { "inor_type\030\001 \001(\0162\021.common.MinorType\022\036\n\004mo" + "de\030\002 \001(\0162\020.common.DataMode\022\r\n\005width\030\003 \001(" + "\005\022\021\n\tprecision\030\004 \001(\005\022\r\n\005scale\030\005 \001(\005\022\020\n\010t" + - "imeZone\030\006 \001(\005*\212\004\n\tMinorType\022\010\n\004LATE\020\000\022\007\n" + + "imeZone\030\006 \001(\005*\225\004\n\tMinorType\022\010\n\004LATE\020\000\022\007\n" + "\003MAP\020\001\022\013\n\007TINYINT\020\003\022\014\n\010SMALLINT\020\004\022\007\n\003INT" + "\020\005\022\n\n\006BIGINT\020\006\022\014\n\010DECIMAL9\020\007\022\r\n\tDECIMAL1" + "8\020\010\022\023\n\017DECIMAL28SPARSE\020\t\022\023\n\017DECIMAL38SPA" + @@ -1793,9 +1802,10 @@ public Builder clearTimeZone() { "\036\022\t\n\005UINT4\020\037\022\t\n\005UINT8\020 \022\022\n\016DECIMAL28DENS" + "E\020!\022\022\n\016DECIMAL38DENSE\020\"\022\010\n\004NULL\020%\022\020\n\014INT" + "ERVALYEAR\020&\022\017\n\013INTERVALDAY\020\'\022\010\n\004LIST\020(\022\022" + - "\n\016GENERIC_OBJECT\020)*4\n\010DataMode\022\014\n\010OPTION" + - "AL\020\000\022\014\n\010REQUIRED\020\001\022\014\n\010REPEATED\020\002B-\n\035org." + - "apache.drill.common.typesB\nTypeProtosH\001" + "\n\016GENERIC_OBJECT\020)\022\t\n\005UNION\020**4\n\010DataMod" + + "e\022\014\n\010OPTIONAL\020\000\022\014\n\010REQUIRED\020\001\022\014\n\010REPEATE" + + "D\020\002B-\n\035org.apache.drill.common.typesB\nTy", + "peProtosH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { diff --git a/protocol/src/main/protobuf/Types.proto b/protocol/src/main/protobuf/Types.proto index d93bcb53674..36899f6502b 100644 --- a/protocol/src/main/protobuf/Types.proto +++ b/protocol/src/main/protobuf/Types.proto @@ -64,6 +64,7 @@ enum MinorType { INTERVALDAY = 39; // Interval type specifying DAY to SECONDS LIST = 40; GENERIC_OBJECT = 41; + UNION = 42; } message MajorType { From 20a7bc2e2d458a9b75befb874746002858001e6c Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Sun, 4 Oct 2015 21:32:34 -0700 Subject: [PATCH 2/5] DRILL-3233: Expression handling for Union types --- .../expression/ExpressionStringBuilder.java | 6 + .../expression/LogicalExpressionBase.java | 7 + .../org/apache/drill/common/types/Types.java | 7 + .../exec/store/mongo/MongoRecordReader.java | 4 +- .../codegen/templates/UnionFunctions.java | 16 +- .../main/codegen/templates/UnionVector.java | 38 ++- .../org/apache/drill/exec/ExecConstants.java | 2 +- .../apache/drill/exec/expr/CloneVisitor.java | 194 +++++++++++++ .../exec/expr/ExpressionTreeMaterializer.java | 119 +++++++- .../drill/exec/expr/fn/DrillFuncHolder.java | 16 ++ .../expr/fn/FunctionGenerationHelper.java | 10 +- .../exec/expr/fn/impl/UnionFunctions.java | 4 +- .../physical/impl/aggregate/HashAggBatch.java | 5 + .../impl/aggregate/StreamingAggBatch.java | 4 + .../impl/project/ProjectRecordBatch.java | 1 + .../exec/record/SimpleVectorWrapper.java | 18 +- .../org/apache/drill/TestExampleQueries.java | 10 + .../apache/drill/common/types/MajorType.java | 32 +++ .../drill/common/types/SchemaTypeProtos.java | 7 + .../apache/drill/common/types/TypeProtos.java | 265 ++++++++++++++++-- protocol/src/main/protobuf/Types.proto | 1 + 21 files changed, 726 insertions(+), 40 deletions(-) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 9791d390fdf..f301b6e3107 100644 --- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -324,4 +324,10 @@ public Void visitNullExpression(NullExpression e, StringBuilder sb) throws Runti return null; } + @Override + public Void visitUnknown(LogicalExpression e, StringBuilder sb) { + sb.append(e.toString()); + return null; + } + } diff --git a/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java b/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java index b8c5b0f9e9a..7dfe4a2ec4a 100644 --- a/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java +++ b/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java @@ -17,12 +17,19 @@ */ package org.apache.drill.common.expression; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.types.TypeProtos.MajorType; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + @JsonPropertyOrder({ "type" }) public abstract class LogicalExpressionBase implements LogicalExpression { diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java index adc3eef49e2..0c468dcb6b8 100644 --- a/common/src/main/java/org/apache/drill/common/types/Types.java +++ b/common/src/main/java/org/apache/drill/common/types/Types.java @@ -34,6 +34,13 @@ public class Types { public static final MajorType REQUIRED_BIT = required(MinorType.BIT); public static final MajorType OPTIONAL_BIT = optional(MinorType.BIT); + public static boolean isUnion(MajorType toType) { + if (toType.getMinorType() == MinorType.UNION) { + return true; + } + return false; + } + public static enum Comparability { UNKNOWN, NONE, EQUAL, ORDERED; } diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java index a9ee98e0720..4aa9aaa1e1d 100644 --- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java +++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java @@ -70,6 +70,7 @@ public class MongoRecordReader extends AbstractRecordReader { private final boolean enableAllTextMode; private final boolean readNumbersAsDouble; + private boolean unionEnabled; public MongoRecordReader( MongoSubScan.MongoSubScanSpec subScanSpec, @@ -140,12 +141,13 @@ private void init(MongoSubScan.MongoSubScanSpec subScanSpec) { MongoClient client = plugin.getClient(addresses); MongoDatabase db = client.getDatabase(subScanSpec.getDbName()); collection = db.getCollection(subScanSpec.getCollectionName()); + this.unionEnabled = fragmentContext.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE); } @Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { this.operatorContext = context; - this.writer = new VectorContainerWriter(output); + this.writer = new VectorContainerWriter(output, unionEnabled); this.jsonReader = new JsonReader(fragmentContext.getManagedBuffer(), Lists.newArrayList(getColumns()), enableAllTextMode, false, readNumbersAsDouble); } diff --git a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java index 41b6b002fa8..8293d8a6892 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java +++ b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java @@ -49,7 +49,21 @@ public class UnionFunctions { <#if !minor.class?starts_with("Decimal")> @SuppressWarnings("unused") - @FunctionTemplate(name = "as${name}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "is${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class UnionIs${name} implements DrillSimpleFunc { + + @Param UnionHolder in; + @Output BitHolder out; + + public void setup() {} + + public void eval() { + out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.${name?upper_case} ? 1 : 0; + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(name = "as${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class CastUnion${name} implements DrillSimpleFunc { @Param UnionHolder in; diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java index 6a72757cc85..bbd09c22c24 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionVector.java +++ b/exec/java-exec/src/main/codegen/templates/UnionVector.java @@ -60,6 +60,9 @@ public class UnionVector implements ValueVector { private State state = State.INIT; private int singleType = 0; private ValueVector singleVector; + private MajorType majorType; + + private final CallBack callBack; private enum State { INIT, SINGLE, MULTI @@ -72,6 +75,8 @@ public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack internalMapWriter = new SingleMapWriter(internalMap, null, true, true); this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class); this.field.addChild(internalMap.getField().clone()); + this.majorType = Types.optional(MinorType.UNION); + this.callBack = callBack; } private void updateState(ValueVector v) { @@ -85,6 +90,17 @@ private void updateState(ValueVector v) { } } + public List getSubTypes() { + return majorType.getSubTypeList(); + } + + private void addSubType(MinorType type) { + majorType = MajorType.newBuilder(this.majorType).addSubType(type).build(); + if (callBack != null) { + callBack.doWork(); + } + } + public boolean isSingleType() { return state == State.SINGLE && singleType != MinorType.LIST_VALUE; } @@ -100,6 +116,7 @@ public MapVector getMap() { int vectorCount = internalMap.size(); mapVector = internalMap.addOrGet("map", Types.optional(MinorType.MAP), MapVector.class); updateState(mapVector); + addSubType(MinorType.MAP); if (internalMap.size() > vectorCount) { mapVector.allocateNew(); } @@ -119,6 +136,7 @@ public MapVector getMap() { int vectorCount = internalMap.size(); ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", Types.optional(MinorType.${name?upper_case}), Nullable${name}Vector.class); updateState(${uncappedName}Vector); + addSubType(MinorType.${name?upper_case}); if (internalMap.size() > vectorCount) { ${uncappedName}Vector.allocateNew(); } @@ -135,6 +153,7 @@ public ListVector getList() { int vectorCount = internalMap.size(); listVector = internalMap.addOrGet("list", Types.optional(MinorType.LIST), ListVector.class); updateState(listVector); + addSubType(MinorType.LIST); if (internalMap.size() > vectorCount) { listVector.allocateNew(); } @@ -210,6 +229,7 @@ public TransferPair makeTransferPair(ValueVector target) { public void transferTo(UnionVector target) { internalMap.makeTransferPair(target.internalMap).transfer(); target.valueCount = valueCount; + target.majorType = majorType; } public void copyFrom(int inIndex, int outIndex, UnionVector from) { @@ -297,6 +317,20 @@ public int getBufferSize() { return internalMap.getBufferSize(); } + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } + + long bufferSize = 0; + for (final ValueVector v : (Iterable) this) { + bufferSize += v.getBufferSizeFor(valueCount); + } + + return (int) bufferSize; + } + @Override public DrillBuf[] getBuffers(boolean clear) { return internalMap.getBuffers(clear); @@ -311,7 +345,9 @@ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { @Override public Iterator iterator() { - return null; + List vectors = Lists.newArrayList(internalMap.iterator()); + vectors.add(typeVector); + return vectors.iterator(); } public class Accessor extends BaseValueVector.BaseAccessor { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index f85a776b907..45d03044dbb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -160,7 +160,7 @@ public interface ExecConstants { public static String MONGO_READER_READ_NUMBERS_AS_DOUBLE = "store.mongo.read_numbers_as_double"; public static OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE, false); - public static BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", false); + public static BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", true); // TODO: We need to add a feature that enables storage plugins to add their own options. Currently we have to declare // in core which is not right. Move this option and above two mongo plugin related options once we have the feature. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java new file mode 100644 index 00000000000..e4fa260324f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr; + +import com.google.common.collect.Lists; +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.CastExpression; +import org.apache.drill.common.expression.ConvertExpression; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.FunctionHolderExpression; +import org.apache.drill.common.expression.IfExpression; +import org.apache.drill.common.expression.IfExpression.IfCondition; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.NullExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedNullConstant; +import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; +import org.apache.drill.common.expression.ValueExpressions.DateExpression; +import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression; +import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; +import org.apache.drill.common.expression.ValueExpressions.FloatExpression; +import org.apache.drill.common.expression.ValueExpressions.IntExpression; +import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression; +import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; +import org.apache.drill.common.expression.ValueExpressions.QuotedString; +import org.apache.drill.common.expression.ValueExpressions.TimeExpression; +import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.drill.exec.expr.fn.DrillFuncHolder; + +import java.util.List; + +public class CloneVisitor extends AbstractExprVisitor { + @Override + public LogicalExpression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { + List args = Lists.newArrayList(); + for (LogicalExpression arg : call.args) { + args.add(arg.accept(this, null)); + } + + return new FunctionCall(call.getName(), args, call.getPosition()); + } + + @Override + public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holder, Void value) throws RuntimeException { + if (holder instanceof DrillFuncHolderExpr) { + List args = Lists.newArrayList(); + for (LogicalExpression arg : holder.args) { + args.add(arg.accept(this, null)); + } + return new DrillFuncHolderExpr(holder.getName(), (DrillFuncHolder) holder.getHolder(), args, holder.getPosition()); + } + return null; + } + + @Override + public LogicalExpression visitIfExpression(IfExpression ifExpr, Void value) throws RuntimeException { + LogicalExpression ifCondition = ifExpr.ifCondition.condition.accept(this, null); + LogicalExpression ifExpression = ifExpr.ifCondition.expression.accept(this, null); + LogicalExpression elseExpression = ifExpr.elseExpression.accept(this, null); + IfExpression.IfCondition condition = new IfCondition(ifCondition, ifExpression); + return IfExpression.newBuilder().setIfCondition(condition).setElse(elseExpression).build(); + } + + @Override + public LogicalExpression visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException { + return visitUnknown(op, value); + } + + @Override + public LogicalExpression visitSchemaPath(SchemaPath path, Void value) throws RuntimeException { + return path; + } + + @Override + public LogicalExpression visitFloatConstant(FloatExpression fExpr, Void value) throws RuntimeException { + return visitUnknown(fExpr, value); + } + + @Override + public LogicalExpression visitIntConstant(IntExpression intExpr, Void value) throws RuntimeException { + return new IntExpression(intExpr.getInt(), intExpr.getPosition()); + } + + @Override + public LogicalExpression visitLongConstant(LongExpression intExpr, Void value) throws RuntimeException { + return visitUnknown(intExpr, value); + } + + + @Override + public LogicalExpression visitDecimal9Constant(Decimal9Expression decExpr, Void value) throws RuntimeException { + return visitUnknown(decExpr, value); + } + + @Override + public LogicalExpression visitDecimal18Constant(Decimal18Expression decExpr, Void value) throws RuntimeException { + return visitUnknown(decExpr, value); + } + + @Override + public LogicalExpression visitDecimal28Constant(Decimal28Expression decExpr, Void value) throws RuntimeException { + return visitUnknown(decExpr, value); + } + + @Override + public LogicalExpression visitDecimal38Constant(Decimal38Expression decExpr, Void value) throws RuntimeException { + return visitUnknown(decExpr, value); + } + + @Override + public LogicalExpression visitDateConstant(DateExpression intExpr, Void value) throws RuntimeException { + return visitUnknown(intExpr, value); + } + + @Override + public LogicalExpression visitTimeConstant(TimeExpression intExpr, Void value) throws RuntimeException { + return visitUnknown(intExpr, value); + } + + @Override + public LogicalExpression visitTimeStampConstant(TimeStampExpression intExpr, Void value) throws RuntimeException { + return visitUnknown(intExpr, value); + } + + @Override + public LogicalExpression visitIntervalYearConstant(IntervalYearExpression intExpr, Void value) throws RuntimeException { + return visitUnknown(intExpr, value); + } + + @Override + public LogicalExpression visitIntervalDayConstant(IntervalDayExpression intExpr, Void value) throws RuntimeException { + return visitUnknown(intExpr, value); + } + + @Override + public LogicalExpression visitDoubleConstant(DoubleExpression dExpr, Void value) throws RuntimeException { + return visitUnknown(dExpr, value); + } + + @Override + public LogicalExpression visitBooleanConstant(BooleanExpression e, Void value) throws RuntimeException { + return visitUnknown(e, value); + } + + @Override + public LogicalExpression visitQuotedStringConstant(QuotedString e, Void value) throws RuntimeException { + return visitUnknown(e, value); + } + + @Override + public LogicalExpression visitCastExpression(CastExpression e, Void value) throws RuntimeException { + return visitUnknown(e, value); + } + + @Override + public LogicalExpression visitConvertExpression(ConvertExpression e, Void value) throws RuntimeException { + return visitUnknown(e, value); + } + + @Override + public LogicalExpression visitNullConstant(TypedNullConstant e, Void value) throws RuntimeException { + return visitUnknown(e, value); + } + + @Override + public LogicalExpression visitNullExpression(NullExpression e, Void value) throws RuntimeException { + return visitUnknown(e, value); + } + + @Override + public LogicalExpression visitUnknown(LogicalExpression e, Void value) throws RuntimeException { + return e; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index bc6d807a611..621fb0e0443 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -17,9 +17,16 @@ */ package org.apache.drill.exec.expr; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Queue; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.base.Preconditions; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.CastExpression; @@ -30,7 +37,9 @@ import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.IfExpression; +import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.LogicalExpressionBase; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.TypedNullConstant; @@ -136,7 +145,7 @@ public static LogicalExpression addCastExpression(LogicalExpression fromExpr, Ma List castArgs = Lists.newArrayList(); castArgs.add(fromExpr); //input_expr - if (!Types.isFixedWidthType(toType)) { + if (!Types.isFixedWidthType(toType) && !Types.isUnion(toType)) { /* We are implicitly casting to VARCHAR so we don't have a max length, * using an arbitrary value. We trim down the size of the stored bytes @@ -241,6 +250,10 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionLookupCont //replace with a new function call, since its argument could be changed. call = new FunctionCall(call.getName(), args, call.getPosition()); + if (hasUnionInput(call)) { + return rewriteUnionFunction(call, functionLookupContext); + } + FunctionResolver resolver = FunctionResolverFactory.getResolver(call); DrillFuncHolder matchedFuncHolder = functionLookupContext.findDrillFunction(resolver, call); @@ -314,6 +327,74 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionLookupCont return NullExpression.INSTANCE; } + private boolean hasUnionInput(FunctionCall call) { + for (LogicalExpression arg : call.args) { + if (arg.getMajorType().getMinorType() == MinorType.UNION) { + if (!call.getName().toLowerCase().startsWith("as") && !call.getName().toLowerCase().startsWith("is") && !call.getName().toLowerCase().startsWith("typeof")) { + return true; + } + } + } + return false; + } + + private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookupContext functionLookupContext) { + LogicalExpression[] args = new LogicalExpression[call.args.size()]; + call.args.toArray(args); + + for (int i = 0; i < args.length; i++) { + LogicalExpression arg = call.args.get(i); + MajorType majorType = arg.getMajorType(); + + if (majorType.getMinorType() != MinorType.UNION) { + continue; + } + + List subTypes = majorType.getSubTypeList(); + Preconditions.checkState(subTypes.size() > 0, "Union type has no subtypes"); + + Queue ifConditions = Lists.newLinkedList(); + + for (MinorType minorType : subTypes) { + LogicalExpression ifCondition = getIsTypeExpressionForType(minorType, arg.accept(new CloneVisitor(), null)); + args[i] = getUnionCastExpressionForType(minorType, arg.accept(new CloneVisitor(), null)); + + List newArgs = Lists.newArrayList(); + for (LogicalExpression e : args) { + newArgs.add(e.accept(new CloneVisitor(), null)); + } + + LogicalExpression thenExpression = new FunctionCall(call.getName(), newArgs, call.getPosition()); + IfExpression.IfCondition condition = new IfCondition(ifCondition, thenExpression); + ifConditions.add(condition); + } + + LogicalExpression ifExpression = ifConditions.poll().expression; + + while (!ifConditions.isEmpty()) { + ifExpression = IfExpression.newBuilder().setIfCondition(ifConditions.poll()).setElse(ifExpression).build(); + } + + args[i] = ifExpression; + return ifExpression.accept(this, functionLookupContext); + } + throw new UnsupportedOperationException("Did not find any Union input types"); + } + + private LogicalExpression getUnionCastExpressionForType(MinorType type, LogicalExpression arg) { + String castFuncName = String.format("as%s", type.toString()); + List args = Lists.newArrayList(); + args.add(arg); + return new FunctionCall(castFuncName, args, ExpressionPosition.UNKNOWN); + } + + private LogicalExpression getIsTypeExpressionForType(MinorType type, LogicalExpression arg) { + String isFuncName = String.format("is%s", type.toString()); + List args = Lists.newArrayList(); + args.add(arg); + return new FunctionCall(isFuncName, args, ExpressionPosition.UNKNOWN); + } + @Override public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionLookupContext functionLookupContext) { IfExpression.IfCondition conditions = ifExpr.ifCondition; @@ -328,19 +409,35 @@ public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionLookupCo if (unionTypeEnabled) { if (thenType != elseType && !(thenType == MinorType.NULL || elseType == MinorType.NULL)) { - MinorType leastRestrictive = TypeCastRules.getLeastRestrictiveType((Arrays.asList(thenType, elseType))); - if (leastRestrictive != thenType) { + MinorType leastRestrictive = MinorType.UNION; + MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL); + if (thenType == MinorType.UNION) { + for (MinorType subType : conditions.expression.getMajorType().getSubTypeList()) { + builder.addSubType(subType); + } + } else { + builder.addSubType(thenType); + } + if (elseType == MinorType.UNION) { + for (MinorType subType : newElseExpr.getMajorType().getSubTypeList()) { + builder.addSubType(subType); + } + } else { + builder.addSubType(elseType); + } + MajorType unionType = builder.build(); +// if (leastRestrictive != thenType) { // Implicitly cast the then expression conditions = new IfExpression.IfCondition(newCondition, - addCastExpression(conditions.expression, Types.optional(MinorType.UNION), functionLookupContext, errorCollector)); - } else if (leastRestrictive != elseType) { + addCastExpression(conditions.expression, unionType, functionLookupContext, errorCollector)); +// } if (leastRestrictive != elseType) { // Implicitly cast the else expression - newElseExpr = addCastExpression(newElseExpr, Types.optional(MinorType.UNION), functionLookupContext, errorCollector); - } else { - conditions = new IfExpression.IfCondition(newCondition, - addCastExpression(conditions.expression, Types.optional(MinorType.UNION), functionLookupContext, errorCollector)); - newElseExpr = addCastExpression(newElseExpr, Types.optional(MinorType.UNION), functionLookupContext, errorCollector); - } + newElseExpr = addCastExpression(newElseExpr, unionType, functionLookupContext, errorCollector); +// } else { +// conditions = new IfExpression.IfCondition(newCondition, +// addCastExpression(conditions.expression, Types.optional(MinorType.UNION), functionLookupContext, errorCollector)); +// newElseExpr = addCastExpression(newElseExpr, Types.optional(MinorType.UNION), functionLookupContext, errorCollector); +// } } } else { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java index 1ec9d69456d..a9cdbc7ea83 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java @@ -19,13 +19,18 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Set; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; @@ -260,6 +265,17 @@ public boolean isFieldReader(int i) { } public MajorType getReturnType(List args) { + if (returnValue.type.getMinorType() == MinorType.UNION) { + Set subTypes = Sets.newHashSet(); + for (ValueReference ref : parameters) { + subTypes.add(ref.getType().getMinorType()); + } + MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL); + for (MinorType subType : subTypes) { + builder.addSubType(subType); + } + return builder.build(); + } if (nullHandling == NullHandling.NULL_IF_NULL) { // if any one of the input types is nullable, then return nullable return type for (LogicalExpression e : args) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java index bf1e252b67e..86734c8a362 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java @@ -55,7 +55,9 @@ public static FunctionHolderExpression getOrderingComparator( null_high ? COMPARE_TO_NULLS_HIGH : COMPARE_TO_NULLS_LOW; if ( ! isComparableType(left.getMajorType() ) - || ! isComparableType(right.getMajorType() ) ) { + || ! isComparableType(right.getMajorType() ) + || isUnionType(left.getMajorType()) + || isUnionType(right.getMajorType()) ) { throw new UnsupportedOperationException( formatCanNotCompareMsg(left.getMajorType(), right.getMajorType())); } @@ -63,6 +65,10 @@ public static FunctionHolderExpression getOrderingComparator( registry, left, right); } + private static boolean isUnionType(MajorType majorType) { + return majorType.getMinorType() == MinorType.UNION; + } + /** * Finds ordering comparator ("compare_to...") FunctionHolderExpression with * a "NULL high" ordering (and considering NULLS equal). @@ -127,7 +133,7 @@ protected static boolean isComparableType(MajorType type) { private static String formatCanNotCompareMsg(MajorType left, MajorType right) { StringBuilder sb = new StringBuilder(); - sb.append("Map, Array or repeated scalar type should not be used in group by, order by or in a comparison operator. Drill does not support compare between "); + sb.append("Map, Array, Union or repeated scalar type should not be used in group by, order by or in a comparison operator. Drill does not support compare between "); appendType(left, sb); sb.append(" and "); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java index 23a5004e9a3..a4b3cc4dbee 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.expr.annotations.Output; import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.NullableIntHolder; import org.apache.drill.exec.expr.holders.UnionHolder; import org.apache.drill.exec.expr.holders.IntHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; @@ -86,13 +87,14 @@ public static class GetType implements DrillSimpleFunc { @Param FieldReader input; @Output - IntHolder out; + NullableIntHolder out; public void setup() {} public void eval() { out.value = input.isSet() ? input.getType().getMinorType().getNumber() : 0; + out.isSet = 1; } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java index a033a8ea050..e2052464833 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java @@ -23,6 +23,7 @@ import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.ErrorCollector; import org.apache.drill.common.expression.ErrorCollectorImpl; +import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.logical.data.NamedExpression; import org.apache.drill.exec.ExecConstants; @@ -211,6 +212,10 @@ private HashAggregator createAggregatorInternal() throws SchemaChangeException, final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry()); + if (expr instanceof IfExpression) { + throw new SchemaChangeException("Union type not supported in aggregate functions"); + } + if (collector.hasErrors()) { throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString()); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java index 2ab1e661df6..9f92d22d056 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java @@ -23,6 +23,7 @@ import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.ErrorCollector; import org.apache.drill.common.expression.ErrorCollectorImpl; +import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.logical.data.NamedExpression; import org.apache.drill.common.types.TypeProtos; @@ -280,6 +281,9 @@ private StreamingAggregator createAggregatorInternal() throws SchemaChangeExcept for (int i = 0; i < valueExprs.length; i++) { final NamedExpression ne = popConfig.getExprs()[i]; final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry()); + if (expr instanceof IfExpression) { + throw new SchemaChangeException("Union type not supported in aggregate functions"); + } if (expr == null) { continue; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index dfca892ff06..2bffded612d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -26,6 +26,7 @@ import org.apache.drill.common.expression.ErrorCollector; import org.apache.drill.common.expression.ErrorCollectorImpl; import org.apache.drill.common.expression.ExpressionPosition; +import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.FunctionCallFactory; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java index 21b3adc9a91..59246ef1086 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java @@ -17,11 +17,14 @@ */ package org.apache.drill.exec.record; +import com.google.common.collect.Lists; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.types.MinorType; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MajorTypeOrBuilder; +import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.complex.AbstractContainerVector; @@ -29,6 +32,9 @@ import org.apache.drill.exec.vector.complex.MapVector; import org.apache.drill.exec.vector.complex.impl.UnionVector; +import java.util.ArrayList; +import java.util.List; + public class SimpleVectorWrapper implements VectorWrapper{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleVectorWrapper.class); @@ -110,8 +116,14 @@ public TypedFieldId getFieldIdIfMatches(int id, SchemaPath expectedPath) { if (v instanceof UnionVector) { TypedFieldId.Builder builder = TypedFieldId.newBuilder(); builder.addId(id).remainder(expectedPath.getRootSegment().getChild()); - builder.finalType(Types.optional(TypeProtos.MinorType.UNION)); - builder.intermediateType(Types.optional(TypeProtos.MinorType.UNION)); + List minorTypes = ((UnionVector) v).getSubTypes(); + MajorType.Builder majorTypeBuilder = MajorType.newBuilder().setMinorType(MinorType.UNION); + for (MinorType type : minorTypes) { + majorTypeBuilder.addSubType(type); + } + MajorType majorType = majorTypeBuilder.build(); + builder.finalType(majorType); + builder.intermediateType(majorType); return builder.build(); } else if (v instanceof AbstractContainerVector) { diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index eebffc11742..e952bff96bd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -32,6 +32,16 @@ public class TestExampleQueries extends BaseTestQuery { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class); + @Test + public void q() throws Exception { + testNoResult("use dfs.tmp"); +// test("select 1 + cast(a as bigint) from t"); +// test("select cast(1 as bigint) + case typeOf(a) when type('bigint') then asBigInt(a) when type('varchar') then cast(asVarChar(a) as bigint) end from t"); +// test("select a from t where typeOf(a) = type('bigint')"); + test("select a_b, typeOf(a_b) type from (select a + b as a_b from t3)"); +// test("select sum(a) from t"); + } + @Test // see DRILL-2328 public void testConcatOnNull() throws Exception { try { diff --git a/protocol/src/main/java/org/apache/drill/common/types/MajorType.java b/protocol/src/main/java/org/apache/drill/common/types/MajorType.java index 29501b83c7d..e8f1cf5cbc0 100644 --- a/protocol/src/main/java/org/apache/drill/common/types/MajorType.java +++ b/protocol/src/main/java/org/apache/drill/common/types/MajorType.java @@ -24,6 +24,8 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.util.ArrayList; +import java.util.List; import com.dyuproject.protostuff.GraphIOUtil; import com.dyuproject.protostuff.Input; @@ -53,6 +55,7 @@ public static MajorType getDefaultInstance() private int precision; private int scale; private int timeZone; + private List subType; public MajorType() { @@ -139,6 +142,19 @@ public MajorType setTimeZone(int timeZone) return this; } + // subType + + public List getSubTypeList() + { + return subType; + } + + public MajorType setSubTypeList(List subType) + { + this.subType = subType; + return this; + } + // java serialization public void readExternal(ObjectInput in) throws IOException @@ -211,6 +227,11 @@ public void mergeFrom(Input input, MajorType message) throws IOException case 6: message.timeZone = input.readInt32(); break; + case 7: + if(message.subType == null) + message.subType = new ArrayList(); + message.subType.add(MinorType.valueOf(input.readEnum())); + break; default: input.handleUnknownField(number, this); } @@ -237,6 +258,15 @@ public void writeTo(Output output, MajorType message) throws IOException if(message.timeZone != 0) output.writeInt32(6, message.timeZone, false); + + if(message.subType != null) + { + for(MinorType subType : message.subType) + { + if(subType != null) + output.writeEnum(7, subType.number, true); + } + } } public String getFieldName(int number) @@ -249,6 +279,7 @@ public String getFieldName(int number) case 4: return "precision"; case 5: return "scale"; case 6: return "timeZone"; + case 7: return "subType"; default: return null; } } @@ -268,6 +299,7 @@ public int getFieldNumber(String name) __fieldMap.put("precision", 4); __fieldMap.put("scale", 5); __fieldMap.put("timeZone", 6); + __fieldMap.put("subType", 7); } } diff --git a/protocol/src/main/java/org/apache/drill/common/types/SchemaTypeProtos.java b/protocol/src/main/java/org/apache/drill/common/types/SchemaTypeProtos.java index 92a7f51fa7a..1a0c4d1e06c 100644 --- a/protocol/src/main/java/org/apache/drill/common/types/SchemaTypeProtos.java +++ b/protocol/src/main/java/org/apache/drill/common/types/SchemaTypeProtos.java @@ -47,6 +47,8 @@ public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.co output.writeInt32(5, message.getScale(), false); if(message.hasTimeZone()) output.writeInt32(6, message.getTimeZone(), false); + for(org.apache.drill.common.types.TypeProtos.MinorType subType : message.getSubTypeList()) + output.writeEnum(7, subType.getNumber(), true); } public boolean isInitialized(org.apache.drill.common.types.TypeProtos.MajorType message) { @@ -104,6 +106,9 @@ public void mergeFrom(com.dyuproject.protostuff.Input input, org.apache.drill.co case 6: builder.setTimeZone(input.readInt32()); break; + case 7: + builder.addSubType(org.apache.drill.common.types.TypeProtos.MinorType.valueOf(input.readEnum())); + break; default: input.handleUnknownField(number, this); } @@ -150,6 +155,7 @@ public static java.lang.String getFieldName(int number) case 4: return "precision"; case 5: return "scale"; case 6: return "timeZone"; + case 7: return "subType"; default: return null; } } @@ -167,6 +173,7 @@ public static int getFieldNumber(java.lang.String name) fieldMap.put("precision", 4); fieldMap.put("scale", 5); fieldMap.put("timeZone", 6); + fieldMap.put("subType", 7); } } diff --git a/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java b/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java index be7fe7443d6..07efdbf4d3d 100644 --- a/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java +++ b/protocol/src/main/java/org/apache/drill/common/types/TypeProtos.java @@ -918,6 +918,32 @@ public interface MajorTypeOrBuilder * */ int getTimeZone(); + + // repeated .common.MinorType sub_type = 7; + /** + * repeated .common.MinorType sub_type = 7; + * + * 

+     * used by Union type
+     *
+ */ + java.util.List getSubTypeList(); + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+     * used by Union type
+     *
+ */ + int getSubTypeCount(); + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+     * used by Union type
+     *
+ */ + org.apache.drill.common.types.TypeProtos.MinorType getSubType(int index); } /** * Protobuf type {@code common.MajorType} @@ -1012,6 +1038,39 @@ private MajorType( timeZone_ = input.readInt32(); break; } + case 56: { + int rawValue = input.readEnum(); + org.apache.drill.common.types.TypeProtos.MinorType value = org.apache.drill.common.types.TypeProtos.MinorType.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(7, rawValue); + } else { + if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + subType_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000040; + } + subType_.add(value); + } + break; + } + case 58: { + int length = input.readRawVarint32(); + int oldLimit = input.pushLimit(length); + while(input.getBytesUntilLimit() > 0) { + int rawValue = input.readEnum(); + org.apache.drill.common.types.TypeProtos.MinorType value = org.apache.drill.common.types.TypeProtos.MinorType.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(7, rawValue); + } else { + if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + subType_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000040; + } + subType_.add(value); + } + } + input.popLimit(oldLimit); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -1020,6 +1079,9 @@ private MajorType( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { + if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + subType_ = java.util.Collections.unmodifiableList(subType_); + } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -1180,6 +1242,40 @@ public int getTimeZone() { return timeZone_; } + // repeated .common.MinorType sub_type = 7; + public static final int SUB_TYPE_FIELD_NUMBER = 7; + private java.util.List subType_; + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+     * used by Union type
+     *
+ */ + public java.util.List getSubTypeList() { + return subType_; + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+     * used by Union type
+     *
+ */ + public int getSubTypeCount() { + return subType_.size(); + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+     * used by Union type
+     *
+ */ + public org.apache.drill.common.types.TypeProtos.MinorType getSubType(int index) { + return subType_.get(index); + } + private void initFields() { minorType_ = org.apache.drill.common.types.TypeProtos.MinorType.LATE; mode_ = org.apache.drill.common.types.TypeProtos.DataMode.OPTIONAL; @@ -1187,6 +1283,7 @@ private void initFields() { precision_ = 0; scale_ = 0; timeZone_ = 0; + subType_ = java.util.Collections.emptyList(); } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -1218,6 +1315,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000020) == 0x00000020)) { output.writeInt32(6, timeZone_); } + for (int i = 0; i < subType_.size(); i++) { + output.writeEnum(7, subType_.get(i).getNumber()); + } getUnknownFields().writeTo(output); } @@ -1251,6 +1351,15 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeInt32Size(6, timeZone_); } + { + int dataSize = 0; + for (int i = 0; i < subType_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeEnumSizeNoTag(subType_.get(i).getNumber()); + } + size += dataSize; + size += 1 * subType_.size(); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -1379,6 +1488,8 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000010); timeZone_ = 0; bitField0_ = (bitField0_ & ~0x00000020); + subType_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); return this; } @@ -1431,6 +1542,11 @@ public org.apache.drill.common.types.TypeProtos.MajorType buildPartial() { to_bitField0_ |= 0x00000020; } result.timeZone_ = timeZone_; + if (((bitField0_ & 0x00000040) == 0x00000040)) { + subType_ = java.util.Collections.unmodifiableList(subType_); + bitField0_ = (bitField0_ & ~0x00000040); + } + result.subType_ = subType_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -1465,6 +1581,16 @@ public Builder mergeFrom(org.apache.drill.common.types.TypeProtos.MajorType othe if (other.hasTimeZone()) { setTimeZone(other.getTimeZone()); } + if (!other.subType_.isEmpty()) { + if (subType_.isEmpty()) { + subType_ = other.subType_; + bitField0_ = (bitField0_ & ~0x00000040); + } else { + ensureSubTypeIsMutable(); + subType_.addAll(other.subType_); + } + onChanged(); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -1760,6 +1886,106 @@ public Builder clearTimeZone() { return this; } + // repeated .common.MinorType sub_type = 7; + private java.util.List subType_ = + java.util.Collections.emptyList(); + private void ensureSubTypeIsMutable() { + if (!((bitField0_ & 0x00000040) == 0x00000040)) { + subType_ = new java.util.ArrayList(subType_); + bitField0_ |= 0x00000040; + } + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+       * used by Union type
+       *
+ */ + public java.util.List getSubTypeList() { + return java.util.Collections.unmodifiableList(subType_); + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+       * used by Union type
+       *
+ */ + public int getSubTypeCount() { + return subType_.size(); + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+       * used by Union type
+       *
+ */ + public org.apache.drill.common.types.TypeProtos.MinorType getSubType(int index) { + return subType_.get(index); + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+       * used by Union type
+       *
+ */ + public Builder setSubType( + int index, org.apache.drill.common.types.TypeProtos.MinorType value) { + if (value == null) { + throw new NullPointerException(); + } + ensureSubTypeIsMutable(); + subType_.set(index, value); + onChanged(); + return this; + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+       * used by Union type
+       *
+ */ + public Builder addSubType(org.apache.drill.common.types.TypeProtos.MinorType value) { + if (value == null) { + throw new NullPointerException(); + } + ensureSubTypeIsMutable(); + subType_.add(value); + onChanged(); + return this; + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+       * used by Union type
+       *
+ */ + public Builder addAllSubType( + java.lang.Iterable values) { + ensureSubTypeIsMutable(); + super.addAll(values, subType_); + onChanged(); + return this; + } + /** + * repeated .common.MinorType sub_type = 7; + * + * 
+       * used by Union type
+       *
+ */ + public Builder clearSubType() { + subType_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:common.MajorType) } @@ -1785,27 +2011,28 @@ public Builder clearTimeZone() { descriptor; static { java.lang.String[] descriptorData = { - "\n\013Types.proto\022\006common\"\225\001\n\tMajorType\022%\n\nm" + + "\n\013Types.proto\022\006common\"\272\001\n\tMajorType\022%\n\nm" + "inor_type\030\001 \001(\0162\021.common.MinorType\022\036\n\004mo" + "de\030\002 \001(\0162\020.common.DataMode\022\r\n\005width\030\003 \001(" + "\005\022\021\n\tprecision\030\004 \001(\005\022\r\n\005scale\030\005 \001(\005\022\020\n\010t" + - "imeZone\030\006 \001(\005*\225\004\n\tMinorType\022\010\n\004LATE\020\000\022\007\n" + - "\003MAP\020\001\022\013\n\007TINYINT\020\003\022\014\n\010SMALLINT\020\004\022\007\n\003INT" + - "\020\005\022\n\n\006BIGINT\020\006\022\014\n\010DECIMAL9\020\007\022\r\n\tDECIMAL1" + - "8\020\010\022\023\n\017DECIMAL28SPARSE\020\t\022\023\n\017DECIMAL38SPA" + - "RSE\020\n\022\t\n\005MONEY\020\013\022\010\n\004DATE\020\014\022\010\n\004TIME\020\r\022\n\n\006" + - "TIMETZ\020\016\022\017\n\013TIMESTAMPTZ\020\017\022\r\n\tTIMESTAMP\020\020", - "\022\014\n\010INTERVAL\020\021\022\n\n\006FLOAT4\020\022\022\n\n\006FLOAT8\020\023\022\007" + - "\n\003BIT\020\024\022\r\n\tFIXEDCHAR\020\025\022\017\n\013FIXED16CHAR\020\026\022" + - "\017\n\013FIXEDBINARY\020\027\022\013\n\007VARCHAR\020\030\022\r\n\tVAR16CH" + - "AR\020\031\022\r\n\tVARBINARY\020\032\022\t\n\005UINT1\020\035\022\t\n\005UINT2\020" + - "\036\022\t\n\005UINT4\020\037\022\t\n\005UINT8\020 \022\022\n\016DECIMAL28DENS" + - "E\020!\022\022\n\016DECIMAL38DENSE\020\"\022\010\n\004NULL\020%\022\020\n\014INT" + - "ERVALYEAR\020&\022\017\n\013INTERVALDAY\020\'\022\010\n\004LIST\020(\022\022" + - "\n\016GENERIC_OBJECT\020)\022\t\n\005UNION\020**4\n\010DataMod" + - "e\022\014\n\010OPTIONAL\020\000\022\014\n\010REQUIRED\020\001\022\014\n\010REPEATE" + - "D\020\002B-\n\035org.apache.drill.common.typesB\nTy", - "peProtosH\001" + "imeZone\030\006 \001(\005\022#\n\010sub_type\030\007 \003(\0162\021.common" + + ".MinorType*\225\004\n\tMinorType\022\010\n\004LATE\020\000\022\007\n\003MA" + + "P\020\001\022\013\n\007TINYINT\020\003\022\014\n\010SMALLINT\020\004\022\007\n\003INT\020\005\022" + + "\n\n\006BIGINT\020\006\022\014\n\010DECIMAL9\020\007\022\r\n\tDECIMAL18\020\010" + + "\022\023\n\017DECIMAL28SPARSE\020\t\022\023\n\017DECIMAL38SPARSE" + + "\020\n\022\t\n\005MONEY\020\013\022\010\n\004DATE\020\014\022\010\n\004TIME\020\r\022\n\n\006TIM", + "ETZ\020\016\022\017\n\013TIMESTAMPTZ\020\017\022\r\n\tTIMESTAMP\020\020\022\014\n" + + "\010INTERVAL\020\021\022\n\n\006FLOAT4\020\022\022\n\n\006FLOAT8\020\023\022\007\n\003B" + + "IT\020\024\022\r\n\tFIXEDCHAR\020\025\022\017\n\013FIXED16CHAR\020\026\022\017\n\013" + + "FIXEDBINARY\020\027\022\013\n\007VARCHAR\020\030\022\r\n\tVAR16CHAR\020" + + "\031\022\r\n\tVARBINARY\020\032\022\t\n\005UINT1\020\035\022\t\n\005UINT2\020\036\022\t" + + "\n\005UINT4\020\037\022\t\n\005UINT8\020 \022\022\n\016DECIMAL28DENSE\020!" + + "\022\022\n\016DECIMAL38DENSE\020\"\022\010\n\004NULL\020%\022\020\n\014INTERV" + + "ALYEAR\020&\022\017\n\013INTERVALDAY\020\'\022\010\n\004LIST\020(\022\022\n\016G" + + "ENERIC_OBJECT\020)\022\t\n\005UNION\020**4\n\010DataMode\022\014" + + "\n\010OPTIONAL\020\000\022\014\n\010REQUIRED\020\001\022\014\n\010REPEATED\020\002", + "B-\n\035org.apache.drill.common.typesB\nTypeP" + + "rotosH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -1817,7 +2044,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_common_MajorType_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_common_MajorType_descriptor, - new java.lang.String[] { "MinorType", "Mode", "Width", "Precision", "Scale", "TimeZone", }); + new java.lang.String[] { "MinorType", "Mode", "Width", "Precision", "Scale", "TimeZone", "SubType", }); return null; } }; diff --git a/protocol/src/main/protobuf/Types.proto b/protocol/src/main/protobuf/Types.proto index 36899f6502b..ca88fe99534 100644 --- a/protocol/src/main/protobuf/Types.proto +++ b/protocol/src/main/protobuf/Types.proto @@ -74,6 +74,7 @@ message MajorType { optional int32 precision = 4; // used for decimal types optional int32 scale = 5; // used for decimal types optional int32 timeZone = 6; // used by TimeStamp type + repeated MinorType sub_type = 7; // used by Union type } From b282a372b6fcce3105161d6d85c5dd52f7c30256 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Fri, 9 Oct 2015 12:59:37 -0700 Subject: [PATCH 3/5] DRILL-3232: Promotable writer --- .../drill/common/expression/IfExpression.java | 17 ++ .../templates/AbstractFieldReader.java | 10 +- .../AbstractPromotableFieldWriter.java | 142 +++++++++++++ .../main/codegen/templates/BaseReader.java | 1 + .../main/codegen/templates/MapWriters.java | 10 +- .../main/codegen/templates/TypeHelper.java | 4 +- .../codegen/templates/UnionListWriter.java | 18 +- .../main/codegen/templates/UnionReader.java | 6 + .../main/codegen/templates/UnionVector.java | 7 +- .../org/apache/drill/exec/ExecConstants.java | 2 +- .../drill/exec/expr/EvaluationVisitor.java | 4 +- .../exec/expr/ExpressionTreeMaterializer.java | 21 +- .../exec/expr/fn/DrillSimpleFuncHolder.java | 11 +- .../exec/expr/fn/impl/UnionFunctions.java | 19 ++ .../IteratorValidatorBatchIterator.java | 2 + .../exec/record/SimpleVectorWrapper.java | 7 + .../drill/exec/record/TypedFieldId.java | 18 +- .../apache/drill/exec/util/BatchPrinter.java | 11 +- .../complex/AbstractContainerVector.java | 4 + .../vector/complex/AbstractMapVector.java | 2 +- .../complex/BaseRepeatedValueVector.java | 5 + .../drill/exec/vector/complex/ListVector.java | 112 ++++++++++- .../complex/impl/AbstractBaseReader.java | 8 + .../vector/complex/impl/PromotableWriter.java | 188 ++++++++++++++++++ .../vector/complex/impl/UnionListReader.java | 29 ++- .../org/apache/drill/TestExampleQueries.java | 17 +- .../drill/exec/store/TestOutputMutator.java | 16 ++ .../complex/writer/TestComplexTypeReader.java | 6 + .../vector/complex/writer/TestJsonReader.java | 18 ++ .../complex/writer/TestPromotableWriter.java | 66 ++++++ .../src/test/resources/jsoninput/union/b.json | 3 + 31 files changed, 716 insertions(+), 68 deletions(-) create mode 100644 exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java create mode 100644 exec/java-exec/src/test/resources/jsoninput/union/b.json diff --git a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java b/common/src/main/java/org/apache/drill/common/expression/IfExpression.java index c0306ce12cd..ab1e4a27044 100644 --- a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java +++ b/common/src/main/java/org/apache/drill/common/expression/IfExpression.java @@ -19,10 +19,13 @@ import java.util.Iterator; import java.util.List; +import java.util.Set; +import com.google.common.collect.Sets; import org.apache.drill.common.expression.visitors.ExprVisitor; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,6 +95,20 @@ public MajorType getMajorType() { // If the return type of one of the "then" expression or "else" expression is nullable, return "if" expression // type as nullable MajorType majorType = elseExpression.getMajorType(); + if (majorType.getMinorType() == MinorType.UNION) { + Set subtypes = Sets.newHashSet(); + for (MinorType subtype : majorType.getSubTypeList()) { + subtypes.add(subtype); + } + for (MinorType subtype : ifCondition.expression.getMajorType().getSubTypeList()) { + subtypes.add(subtype); + } + MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL); + for (MinorType subtype : subtypes) { + builder.addSubType(subtype); + } + return builder.build(); + } if (majorType.getMode() == DataMode.OPTIONAL) { return majorType; } diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java index 89afd7c5505..a4d22cf08a3 100644 --- a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java +++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java @@ -33,6 +33,10 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe super(); } + public boolean isSet() { + return true; + } + <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", "Character", "DateTime", "Period", "Double", "Float", "Text", "String", "Byte", "Short", "byte[]"] as friendlyType> @@ -57,9 +61,9 @@ public void copyAsValue(MapWriter writer){ public void copyAsField(String name, MapWriter writer){ fail("CopyAsField MapWriter"); } - public void copyAsValue(ListWriter writer){ - fail("CopyAsValueList"); - } +// public void copyAsValue(ListWriter writer){ +// fail("CopyAsValueList"); +// } public void copyAsField(String name, ListWriter writer){ fail("CopyAsFieldList"); } diff --git a/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java new file mode 100644 index 00000000000..f2f81934ce5 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.drill.common.types.TypeProtos.MinorType; + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractPromotableFieldWriter.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> + +/* + * A FieldWriter which delegates calls to another FieldWriter. The delegate FieldWriter can be promoted to a new type + * when necessary. Classes that extend this class are responsible for handling promotion. + * + * This class is generated using freemarker and the ${.template_name} template. + * + */ +@SuppressWarnings("unused") +abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter { + AbstractPromotableFieldWriter(FieldWriter parent) { + super(parent); + } + + /** + * Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type + * @param type + * @return + */ + abstract protected FieldWriter getWriter(MinorType type); + + /** + * Return the current FieldWriter + * @return + */ + abstract protected FieldWriter getWriter(); + + @Override + public void start() { + getWriter(MinorType.MAP).start(); + } + + @Override + public void end() { + getWriter(MinorType.MAP).end(); + } + + @Override + public void startList() { + getWriter(MinorType.LIST).startList(); + } + + @Override + public void endList() { + getWriter(MinorType.LIST).endList(); + } + + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#if !minor.class?starts_with("Decimal") > + @Override + public void write(${name}Holder holder) { + getWriter(MinorType.${name?upper_case}).write(holder); + } + + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, ); + } + + + + + public void writeNull() { + } + + @Override + public MapWriter map() { + return getWriter(MinorType.LIST).map(); + } + + @Override + public ListWriter list() { + return getWriter(MinorType.LIST).list(); + } + + @Override + public MapWriter map(String name) { + return getWriter(MinorType.MAP).map(name); + } + + @Override + public ListWriter list(String name) { + return getWriter(MinorType.MAP).list(name); + } + + <#list vv.types as type><#list type.minor as minor> + <#assign lowerName = minor.class?uncap_first /> + <#if lowerName == "int" ><#assign lowerName = "integer" /> + <#assign upperName = minor.class?upper_case /> + <#assign capName = minor.class?cap_first /> + <#if !minor.class?starts_with("Decimal") > + + @Override + public ${capName}Writer ${lowerName}(String name) { + return getWriter(MinorType.MAP).${lowerName}(name); + } + + @Override + public ${capName}Writer ${lowerName}() { + return getWriter(MinorType.LIST).${lowerName}(); + } + + + + + public void copyReader(FieldReader reader) { + getWriter().copyReader(reader); + } + + public void copyReaderToField(String name, FieldReader reader) { + getWriter().copyReaderToField(name, reader); + } +} diff --git a/exec/java-exec/src/main/codegen/templates/BaseReader.java b/exec/java-exec/src/main/codegen/templates/BaseReader.java index 4fce409ecdd..78f32f4cde0 100644 --- a/exec/java-exec/src/main/codegen/templates/BaseReader.java +++ b/exec/java-exec/src/main/codegen/templates/BaseReader.java @@ -36,6 +36,7 @@ public interface BaseReader extends Positionable{ void read(UnionHolder holder); void read(int index, UnionHolder holder); void copyAsValue(UnionWriter writer); + boolean isSet(); public interface MapReader extends BaseReader, Iterable{ FieldReader reader(String name); diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java index 42cacabd3ee..e6becf2a950 100644 --- a/exec/java-exec/src/main/codegen/templates/MapWriters.java +++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java @@ -85,8 +85,8 @@ public MapWriter map(String name) { MapVector vector=container.addOrGet(name,MapVector.TYPE,MapVector.class); writer=new SingleMapWriter(vector,this); } else { - UnionVector vector = container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class); - writer = new UnionWriter(vector); + MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class); + writer = new PromotableWriter(vector, container); } if(vectorCount != container.size()) { writer.allocate(); @@ -127,7 +127,7 @@ public ListWriter list(String name) { if (!unionEnabled){ writer = new SingleListWriter(name,container,this); } else{ - writer = new UnionWriter(container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class)); + writer = new PromotableWriter(container.addOrGet(name, Types.optional(MinorType.LIST), ListVector.class), container); } if (container.size() > vectorCount) { writer.allocate(); @@ -215,8 +215,8 @@ public void end() { if(writer == null) { ValueVector vector; if (unionEnabled){ - UnionVector v = container.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class); - writer = new UnionWriter(v); + ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class); + writer = new PromotableWriter(v, container); vector = v; } else { ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class); diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java index d9810c79ec9..b8178cb930c 100644 --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java @@ -206,6 +206,7 @@ public static Class getWriterImpl( MinorType type, DataMode mode){ case MAP: switch (mode) { case REQUIRED: + case OPTIONAL: return SingleMapWriter.class; case REPEATED: return RepeatedMapWriter.class; @@ -213,7 +214,8 @@ public static Class getWriterImpl( MinorType type, DataMode mode){ case LIST: switch (mode) { case REQUIRED: - return SingleListWriter.class; + case OPTIONAL: + return UnionListWriter.class; case REPEATED: return RepeatedListWriter.class; } diff --git a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java index fd7256b06ac..35c0bc1bc02 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java +++ b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java @@ -34,9 +34,9 @@ public class UnionListWriter extends AbstractFieldWriter { ListVector vector; - UnionVector data; +// UnionVector data; UInt4Vector offsets; - private UnionWriter writer; + private PromotableWriter writer; private boolean inMap = false; private String mapName; private int lastIndex = 0; @@ -44,11 +44,15 @@ public class UnionListWriter extends AbstractFieldWriter { public UnionListWriter(ListVector vector) { super(null); this.vector = vector; - this.data = (UnionVector) vector.getDataVector(); - this.writer = new UnionWriter(data); +// this.data = (UnionVector) vector.getDataVector(); + this.writer = new PromotableWriter(vector.getDataVector(), vector); this.offsets = vector.getOffsetVector(); } + public UnionListWriter(ListVector vector, AbstractFieldWriter parent) { + this(vector); + } + @Override public void allocate() { vector.allocateNew(); @@ -115,7 +119,7 @@ public ListWriter list() { public ListWriter list(String name) { final int nextOffset = offsets.getAccessor().get(idx() + 1); vector.getMutator().setNotNull(idx()); - data.getMutator().setType(nextOffset, MinorType.MAP); +// data.getMutator().setType(nextOffset, MinorType.MAP); writer.setPosition(nextOffset); ListWriter listWriter = writer.list(name); return listWriter; @@ -142,7 +146,7 @@ public void start() { assert inMap; final int nextOffset = offsets.getAccessor().get(idx() + 1); vector.getMutator().setNotNull(idx()); - data.getMutator().setType(nextOffset, MinorType.MAP); +// data.getMutator().setType(nextOffset, MinorType.MAP); offsets.getMutator().setSafe(idx() + 1, nextOffset); writer.setPosition(nextOffset); } @@ -167,7 +171,7 @@ public void end() { if (inMap) { final int nextOffset = offsets.getAccessor().get(idx() + 1); vector.getMutator().setNotNull(idx()); - data.getMutator().setType(nextOffset, MinorType.MAP); +// data.getMutator().setType(nextOffset, MinorType.MAP); writer.setPosition(nextOffset); ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}(mapName); ${uncappedName}Writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); diff --git a/exec/java-exec/src/main/codegen/templates/UnionReader.java b/exec/java-exec/src/main/codegen/templates/UnionReader.java index 38d424786e9..4943ea0debc 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionReader.java +++ b/exec/java-exec/src/main/codegen/templates/UnionReader.java @@ -156,6 +156,12 @@ public void copyAsValue(${name}Writer writer){ + @Override + public void copyAsValue(ListWriter writer) { + ComplexCopier copier = new ComplexCopier(this, (FieldWriter) writer); + copier.write(); + } + @Override public void setPosition(int index) { super.setPosition(index); diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java index bbd09c22c24..3e7b3bf4290 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionVector.java +++ b/exec/java-exec/src/main/codegen/templates/UnionVector.java @@ -75,7 +75,7 @@ public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack internalMapWriter = new SingleMapWriter(internalMap, null, true, true); this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class); this.field.addChild(internalMap.getField().clone()); - this.majorType = Types.optional(MinorType.UNION); + this.majorType = field.getType(); this.callBack = callBack; } @@ -243,6 +243,11 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { copyFrom(inIndex, outIndex, from); } + public void addVector(ValueVector v) { + internalMap.putChild(v.getField().getType().getMinorType().name().toLowerCase(), v); + addSubType(v.getField().getType().getMinorType()); + } + private class TransferImpl implements TransferPair { UnionVector to; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 45d03044dbb..f85a776b907 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -160,7 +160,7 @@ public interface ExecConstants { public static String MONGO_READER_READ_NUMBERS_AS_DOUBLE = "store.mongo.read_numbers_as_double"; public static OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE, false); - public static BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", true); + public static BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", false); // TODO: We need to add a feature that enables storage plugins to add their own options. Currently we have to declare // in core which is not right. Move this option and above two mongo plugin related options once we have the feature. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index 19aa3c9288a..424174ed657 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -21,6 +21,8 @@ import java.util.List; import java.util.Set; +import com.google.common.collect.Maps; +import com.sun.codemodel.JOp; import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.ConvertExpression; @@ -387,7 +389,7 @@ private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpressio if (seg.isArray()) { // stop once we get to the last segment and the final type is neither complex nor repeated (map, list, repeated list). // In case of non-complex and non-repeated type, we return Holder, in stead of FieldReader. - if (seg.isLastPath() && !complex && !repeated) { + if (seg.isLastPath() && !complex && !repeated && !e.getTypedFieldId().isListVector()) { break; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 621fb0e0443..827623b2094 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -382,6 +382,9 @@ private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookup } private LogicalExpression getUnionCastExpressionForType(MinorType type, LogicalExpression arg) { + if (type == MinorType.UNION) { + return null; + } String castFuncName = String.format("as%s", type.toString()); List args = Lists.newArrayList(); args.add(arg); @@ -406,6 +409,7 @@ public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionLookupCo MinorType thenType = conditions.expression.getMajorType().getMinorType(); MinorType elseType = newElseExpr.getMajorType().getMinorType(); + boolean hasUnion = thenType == MinorType.UNION || elseType == MinorType.UNION; if (unionTypeEnabled) { if (thenType != elseType && !(thenType == MinorType.NULL || elseType == MinorType.NULL)) { @@ -426,18 +430,9 @@ public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionLookupCo builder.addSubType(elseType); } MajorType unionType = builder.build(); -// if (leastRestrictive != thenType) { - // Implicitly cast the then expression - conditions = new IfExpression.IfCondition(newCondition, - addCastExpression(conditions.expression, unionType, functionLookupContext, errorCollector)); -// } if (leastRestrictive != elseType) { - // Implicitly cast the else expression - newElseExpr = addCastExpression(newElseExpr, unionType, functionLookupContext, errorCollector); -// } else { -// conditions = new IfExpression.IfCondition(newCondition, -// addCastExpression(conditions.expression, Types.optional(MinorType.UNION), functionLookupContext, errorCollector)); -// newElseExpr = addCastExpression(newElseExpr, Types.optional(MinorType.UNION), functionLookupContext, errorCollector); -// } + conditions = new IfExpression.IfCondition(newCondition, + addCastExpression(conditions.expression, unionType, functionLookupContext, errorCollector)); + newElseExpr = addCastExpression(newElseExpr, unionType, functionLookupContext, errorCollector); } } else { @@ -492,7 +487,7 @@ public boolean apply(LogicalExpression input) { } } - if (!unionTypeEnabled) { + if (!hasUnion) { // If the type of the IF expression is nullable, apply a convertToNullable*Holder function for "THEN"/"ELSE" // expressions whose type is not nullable. if (IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).build().getMajorType().getMode() diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java index 859f2704ba4..78e4c62ee3d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import com.sun.codemodel.JOp; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; @@ -97,10 +98,16 @@ protected HoldingContainer generateEvalBody(ClassGenerator g, HoldingContaine JExpression e = null; for (HoldingContainer v : inputVariables) { if (v.isOptional()) { + JExpression isNullExpr; + if (v.isReader()) { + isNullExpr = JOp.cond(v.getHolder().invoke("isSet"), JExpr.lit(1), JExpr.lit(0)); + } else { + isNullExpr = v.getIsSet(); + } if (e == null) { - e = v.getIsSet(); + e = isNullExpr; } else { - e = e.mul(v.getIsSet()); + e = e.mul(isNullExpr); } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java index a4b3cc4dbee..34b14fed929 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java @@ -24,6 +24,7 @@ import org.apache.drill.exec.expr.annotations.Output; import org.apache.drill.exec.expr.annotations.Param; import org.apache.drill.exec.expr.holders.NullableIntHolder; +import org.apache.drill.exec.expr.holders.NullableUInt1Holder; import org.apache.drill.exec.expr.holders.UnionHolder; import org.apache.drill.exec.expr.holders.IntHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; @@ -114,4 +115,22 @@ public void eval() { out.isSet = in.isSet() ? 1 : 0; } } + + @SuppressWarnings("unused") + @FunctionTemplate(name = "asList", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class CastUnionList implements DrillSimpleFunc { + + @Param UnionHolder in; + @Output UnionHolder out; + + public void setup() {} + + public void eval() { + if (in.isSet == 1) { + out.reader = in.reader; + } else { + out.isSet = 0; + } + } + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java index efd155e1f64..1687c3d1a95 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java @@ -30,6 +30,7 @@ import org.apache.drill.exec.record.WritableBatch; import org.apache.drill.exec.record.selection.SelectionVector2; import org.apache.drill.exec.record.selection.SelectionVector4; +import org.apache.drill.exec.util.BatchPrinter; import org.apache.drill.exec.vector.VectorValidator; public class IteratorValidatorBatchIterator implements CloseableRecordBatch { @@ -133,6 +134,7 @@ public IterOutcome next() { throw new IllegalStateException (String.format("Incoming batch of %s has size %d, which is beyond the limit of %d", incoming.getClass().getName(), incoming.getRecordCount(), MAX_BATCH_SIZE)); } +// BatchPrinter.printBatch(incoming); if (VALIDATE_VECTORS) { VectorValidator.validate(incoming); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java index 59246ef1086..2a8e91fe2ed 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.complex.AbstractContainerVector; import org.apache.drill.exec.vector.complex.AbstractMapVector; +import org.apache.drill.exec.vector.complex.ListVector; import org.apache.drill.exec.vector.complex.MapVector; import org.apache.drill.exec.vector.complex.impl.UnionVector; @@ -125,6 +126,12 @@ public TypedFieldId getFieldIdIfMatches(int id, SchemaPath expectedPath) { builder.finalType(majorType); builder.intermediateType(majorType); return builder.build(); + } else if (v instanceof ListVector) { + ListVector list = (ListVector) v; + TypedFieldId.Builder builder = TypedFieldId.newBuilder(); + builder.intermediateType(v.getField().getType()); + builder.addId(id); + return list.getFieldIdIfMatches(builder, true, expectedPath.getRootSegment().getChild()); } else if (v instanceof AbstractContainerVector) { // we're looking for a multi path. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java index db8bfa82033..931f981595d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java @@ -34,6 +34,7 @@ public class TypedFieldId { final MajorType intermediateType; final int[] fieldIds; final boolean isHyperReader; + final boolean isListVector; final PathSegment remainder; public TypedFieldId(MajorType type, int... fieldIds) { @@ -49,12 +50,17 @@ public TypedFieldId(MajorType type, boolean isHyper, int... fieldIds) { } public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, PathSegment remainder, int... fieldIds) { + this(intermediateType, secondaryFinal, finalType, isHyper, false, remainder, fieldIds); + } + + public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, boolean isListVector, PathSegment remainder, int... fieldIds) { super(); this.intermediateType = intermediateType; this.finalType = finalType; this.secondaryFinal = secondaryFinal; this.fieldIds = fieldIds; this.isHyperReader = isHyper; + this.isListVector = isListVector; this.remainder = remainder; } @@ -90,6 +96,10 @@ public boolean isHyperReader() { return isHyperReader; } + public boolean isListVector() { + return isListVector; + } + public MajorType getIntermediateType() { return intermediateType; } @@ -122,6 +132,7 @@ public static class Builder{ PathSegment remainder; boolean hyperReader = false; boolean withIndex = false; + boolean isListVector = false; public Builder addId(int id) { ids.add(id); @@ -143,6 +154,11 @@ public Builder hyper() { return this; } + public Builder listVector() { + this.isListVector = true; + return this; + } + public Builder finalType(MajorType finalType) { this.finalType = finalType; return this; @@ -179,7 +195,7 @@ public TypedFieldId build() { // TODO: there is a bug here with some things. //if(intermediateType != finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.OPTIONAL).build(); - return new TypedFieldId(intermediateType, secondaryFinal, actualFinalType, hyperReader, remainder, ids.toArray()); + return new TypedFieldId(intermediateType, secondaryFinal, actualFinalType, hyperReader, isListVector, remainder, ids.toArray()); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java index 6021cd253b9..55173f93a66 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java @@ -75,13 +75,16 @@ public static void printBatch(VectorAccessible batch) { } for (ValueVector vv : vectors) { Object o = vv.getAccessor().getObject(row); + String value; + if (o == null) { + value = "null"; + } else if (o instanceof byte[]) { - String value = new String((byte[]) o); - System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14)); + value = new String((byte[]) o); } else { - String value = o.toString(); - System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14)); + value = o.toString(); } + System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14)); } System.out.printf("|\n"); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java index d14dca6bebf..17b42a133b4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java @@ -22,6 +22,7 @@ import java.util.Collections; import javax.annotation.Nullable; +import javax.validation.constraints.DecimalMin.List; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -155,6 +156,9 @@ public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean ad // we're looking for a multi path. AbstractContainerVector c = (AbstractContainerVector) v; return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); + } else if(v instanceof ListVector) { + ListVector list = (ListVector) v; + return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); } else { if (seg.isNamed()) { if(addToBreadCrumb) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java index 2c93c319f85..1706f5cf036 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java @@ -173,7 +173,7 @@ public T getChild(String name, Class clazz) { * * Note that this method does not enforce any vector type check nor throws a schema change exception. */ - protected void putChild(String name, ValueVector vector) { + public void putChild(String name, ValueVector vector) { putVector(name, vector); field.addChild(vector.getField()); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java index 19b5ab449cb..29f6cda8818 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java @@ -202,6 +202,11 @@ public AddOrGetResult addOrGetVector(VectorDescriptor return new AddOrGetResult<>((T)vector, created); } + protected void replaceDataVector(ValueVector v) { + vector.clear(); + vector = v; + } + public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java index ccd6239ec7b..a44efe343fe 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java @@ -21,6 +21,8 @@ import com.google.common.collect.ObjectArrays; import io.netty.buffer.DrillBuf; import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; import org.apache.drill.exec.memory.BufferAllocator; @@ -28,8 +30,10 @@ import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; +import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.util.CallBack; import org.apache.drill.exec.util.JsonStringArrayList; +import org.apache.drill.exec.vector.AddOrGetResult; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.UInt1Vector; import org.apache.drill.exec.vector.UInt4Vector; @@ -46,20 +50,22 @@ public class ListVector extends BaseRepeatedValueVector { - UInt4Vector offsets; - protected final UInt1Vector bits; - Mutator mutator = new Mutator(); - Accessor accessor = new Accessor(); - UnionListWriter writer; - UnionListReader reader; + private UInt4Vector offsets; + private final UInt1Vector bits; + private Mutator mutator = new Mutator(); + private Accessor accessor = new Accessor(); + private UnionListWriter writer; + private UnionListReader reader; + private CallBack callBack; public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { - super(field, allocator, new UnionVector(field, allocator, callBack)); + super(field, allocator); this.bits = new UInt1Vector(MaterializedField.create("$bits$", Types.required(MinorType.UINT1)), allocator); offsets = getOffsetVector(); this.field.addChild(getDataVector().getField()); this.writer = new UnionListWriter(this); this.reader = new UnionListReader(this); + this.callBack = callBack; } public UnionListWriter getWriter() { @@ -87,8 +93,8 @@ public void copyFrom(int inIndex, int outIndex, ListVector from) { } @Override - public UnionVector getDataVector() { - return (UnionVector) vector; + public ValueVector getDataVector() { + return vector; } @Override @@ -107,10 +113,12 @@ private class TransferImpl implements TransferPair { public TransferImpl(MaterializedField field) { to = new ListVector(field, allocator, null); + to.addOrGetVector(new VectorDescriptor(vector.getField().getType())); } public TransferImpl(ListVector to) { this.to = to; + to.addOrGetVector(new VectorDescriptor(vector.getField().getType())); } @Override @@ -182,6 +190,11 @@ protected UserBitShared.SerializedField.Builder getMetadataBuilder() { .addChild(bits.getMetadata()) .addChild(vector.getMetadata()); } + public AddOrGetResult addOrGetVector(VectorDescriptor descriptor) { + AddOrGetResult result = super.addOrGetVector(descriptor); + reader = new UnionListReader(this); + return result; + } @Override public int getBufferSize() { @@ -232,20 +245,97 @@ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength)); } + public UnionVector promoteToUnion() { + MaterializedField newField = MaterializedField.create(getField().getPath(), Types.optional(MinorType.UNION)); + UnionVector vector = new UnionVector(newField, allocator, null); + replaceDataVector(vector); + reader = new UnionListReader(this); + return vector; + } + + public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) { + if (seg == null) { + if (addToBreadCrumb) { + builder.intermediateType(this.getField().getType()); + } + return builder.finalType(this.getField().getType()).build(); + } + + if (seg.isArray()) { + if (seg.isLastPath()) { + builder // + .withIndex() // + .listVector() + .finalType(getDataVector().getField().getType()); + + // remainder starts with the 1st array segment in SchemaPath. + // only set remainder when it's the only array segment. + if (addToBreadCrumb) { + addToBreadCrumb = false; + builder.remainder(seg); + } + return builder.build(); + } else { + if (addToBreadCrumb) { + addToBreadCrumb = false; + builder.remainder(seg); + } + } + } else { + return null; + } + + ValueVector v = getDataVector(); + if (v instanceof AbstractContainerVector) { + // we're looking for a multi path. + AbstractContainerVector c = (AbstractContainerVector) v; + return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); + } else if (v instanceof ListVector) { + ListVector list = (ListVector) v; + return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); + } else { + if (seg.isNamed()) { + if(addToBreadCrumb) { + builder.intermediateType(v.getField().getType()); + } + builder.finalType(v.getField().getType()); + } else { + builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); + } + + if (seg.isLastPath()) { + return builder.build(); + } else { + PathSegment child = seg.getChild(); + if (child.isLastPath() && child.isArray()) { + if (addToBreadCrumb) { + builder.remainder(child); + } + builder.withIndex(); + builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); + return builder.build(); + } else { +// logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong."); + return null; + } + } + } + } + private int lastSet; public class Accessor extends BaseRepeatedAccessor { @Override public Object getObject(int index) { - if (bits.getAccessor().isNull(index)) { + if (isNull(index)) { return null; } final List vals = new JsonStringArrayList<>(); final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); final int start = offsetsAccessor.get(index); final int end = offsetsAccessor.get(index + 1); - final UnionVector.Accessor valuesAccessor = getDataVector().getAccessor(); + final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor(); for(int i = start; i < end; i++) { vals.add(valuesAccessor.getObject(i)); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java index fea326c4c96..7624b1d642e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java @@ -24,6 +24,8 @@ import org.apache.drill.exec.expr.holders.UnionHolder; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.vector.complex.reader.FieldReader; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; +import org.apache.drill.exec.vector.complex.writer.FieldWriter; abstract class AbstractBaseReader implements FieldReader{ @@ -88,4 +90,10 @@ public void read(int index, UnionHolder holder) { public void copyAsValue(UnionWriter writer) { throw new IllegalStateException("The current reader doesn't support reading union type"); } + + @Override + public void copyAsValue(ListWriter writer) { + ComplexCopier copier = new ComplexCopier(this, (FieldWriter)writer); + copier.write(); + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java new file mode 100644 index 00000000000..d3459c3aed0 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex.impl; + +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.expr.TypeHelper; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.TransferPair; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VectorDescriptor; +import org.apache.drill.exec.vector.ZeroVector; +import org.apache.drill.exec.vector.complex.AbstractMapVector; +import org.apache.drill.exec.vector.complex.ListVector; +import org.apache.drill.exec.vector.complex.writer.FieldWriter; + +import java.lang.reflect.Constructor; + +/** + * This FieldWriter implementation delegates all FieldWriter API calls to an inner FieldWriter. This inner field writer + * can start as a specific type, and this class will promote the writer to a UnionWriter if a call is made that the specifically + * typed writer cannot handle. A new UnionVector is created, wrapping the original vector, and replaces the original vector + * in the parent vector, which can be either an AbstractMapVector or a ListVector. + */ +public class PromotableWriter extends AbstractPromotableFieldWriter { + + private final AbstractMapVector parentContainer; + private final ListVector listVector; + private int position; + + private enum State { + UNTYPED, SINGLE, UNION + } + + private MinorType type; + private ValueVector vector; + private UnionVector unionVector; + private State state; + private FieldWriter writer; + + public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) { + super(null); + this.parentContainer = parentContainer; + this.listVector = null; + init(v); + } + + public PromotableWriter(ValueVector v, ListVector listVector) { + super(null); + this.listVector = listVector; + this.parentContainer = null; + init(v); + } + + private void init(ValueVector v) { + if (v instanceof UnionVector) { + state = State.UNION; + unionVector = (UnionVector) v; + writer = new UnionWriter(unionVector); + } else if (v instanceof ZeroVector) { + state = State.UNTYPED; + } else { + setWriter(v); + } + } + + private void setWriter(ValueVector v) { + state = State.SINGLE; + vector = v; + type = v.getField().getType().getMinorType(); + Class writerClass = TypeHelper.getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode()); + if (writerClass.equals(SingleListWriter.class)) { + writerClass = UnionListWriter.class; + } + Class vectorClass = TypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField().getDataMode()); + try { + Constructor constructor = null; + for (Constructor c : writerClass.getConstructors()) { + if (c.getParameterTypes().length == 4) { + constructor = c; + } + } + if (constructor == null) { + constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class); + writer = (FieldWriter) constructor.newInstance(vector, null); + } else { + writer = (FieldWriter) constructor.newInstance(vector, null, true, false); + } + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + + @Override + public void setPosition(int index) { + super.setPosition(index); + FieldWriter w = getWriter(); + if (w == null) { + position = index; + } else { + w.setPosition(index); + } + } + + protected FieldWriter getWriter(MinorType type) { + if (state == State.UNION) { + return writer; + } + if (state == State.UNTYPED) { + if (type == null) { + return null; + } + ValueVector v = listVector.addOrGetVector(new VectorDescriptor(Types.optional(type))).getVector(); + v.allocateNew(); + setWriter(v); + writer.setPosition(position); + } + if (type != this.type) { + return promoteToUnion(); + } + return writer; + } + + protected FieldWriter getWriter() { + return getWriter(type); + } + + private FieldWriter promoteToUnion() { + String name = vector.getField().getLastName(); + TransferPair tp = vector.getTransferPair(new FieldReference(vector.getField().getType().getMinorType().name().toLowerCase())); + tp.transfer(); + if (parentContainer != null) { + unionVector = parentContainer.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class); + } else if (listVector != null) { + unionVector = listVector.promoteToUnion(); + } + unionVector.addVector(tp.getTo()); + writer = new UnionWriter(unionVector); + writer.setPosition(idx()); + for (int i = 0; i < idx(); i++) { + unionVector.getMutator().setType(i, vector.getField().getType().getMinorType()); + } + vector = null; + state = State.UNION; + return writer; + } + + @Override + public void allocate() { + getWriter().allocate(); + } + + @Override + public void clear() { + getWriter().clear(); + } + + @Override + public MaterializedField getField() { + return getWriter().getField(); + } + + @Override + public int getValueCapacity() { + return getWriter().getValueCapacity(); + } + + @Override + public void close() throws Exception { + getWriter().close(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java index fef33259bd3..db2d8e48512 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java @@ -19,23 +19,28 @@ package org.apache.drill.exec.vector.complex.impl; import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.expr.holders.UnionHolder; import org.apache.drill.exec.vector.UInt4Vector; +import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.complex.ListVector; import org.apache.drill.exec.vector.complex.reader.FieldReader; +import org.apache.drill.exec.vector.complex.writer.BaseWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.drill.exec.vector.complex.writer.FieldWriter; public class UnionListReader extends AbstractFieldReader { private ListVector vector; - private UnionVector data; + private ValueVector data; private UInt4Vector offsets; - private UnionReader reader; public UnionListReader(ListVector vector) { this.vector = vector; this.data = vector.getDataVector(); this.offsets = vector.getOffsetVector(); - this.reader = (UnionReader) data.getReader(); } @Override @@ -43,9 +48,10 @@ public boolean isSet() { return true; } - @Override + MajorType type = Types.optional(MinorType.LIST); + public MajorType getType() { - return reader.getType(); + return type; } private int currentOffset; @@ -60,7 +66,7 @@ public void setPosition(int index) { @Override public FieldReader reader() { - return reader; + return data.getReader(); } @Override @@ -74,17 +80,22 @@ public void read(int index, UnionHolder holder) { for (int i = -1; i < index; i++) { next(); } - holder.reader = reader; - holder.isSet = reader.isSet() ? 1 : 0; + holder.reader = data.getReader(); + holder.isSet = data.getReader().isSet() ? 1 : 0; } @Override public boolean next() { if (currentOffset + 1 < maxOffset) { - reader.setPosition(++currentOffset); + data.getReader().setPosition(++currentOffset); return true; } else { return false; } } + + public void copyAsValue(ListWriter writer) { + ComplexCopier copier = new ComplexCopier(this, (FieldWriter) writer); + copier.write(); + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index e952bff96bd..05bd1387a97 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -26,20 +26,21 @@ import org.apache.drill.common.util.FileUtils; import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.junit.After; import org.junit.Ignore; import org.junit.Test; public class TestExampleQueries extends BaseTestQuery { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class); - @Test - public void q() throws Exception { - testNoResult("use dfs.tmp"); -// test("select 1 + cast(a as bigint) from t"); -// test("select cast(1 as bigint) + case typeOf(a) when type('bigint') then asBigInt(a) when type('varchar') then cast(asVarChar(a) as bigint) end from t"); -// test("select a from t where typeOf(a) = type('bigint')"); - test("select a_b, typeOf(a_b) type from (select a + b as a_b from t3)"); -// test("select sum(a) from t"); + @After + public void reset() throws Exception { + String[] options = new String[] { ExecConstants.SLICE_TARGET, PlannerSettings.HASHJOIN.getOptionName(), PlannerSettings.HASHAGG.getOptionName(), + PlannerSettings.STREAMAGG.getOptionName(), PlannerSettings.MERGEJOIN.getOptionName(), PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR.getOptionName() }; + for (String option : options) { + testNoResult(String.format("reset `%s`", option)); + } } @Test // see DRILL-2328 diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java index 51b909b366e..0c5ff49d5a6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java @@ -17,12 +17,14 @@ */ package org.apache.drill.exec.store; +import com.google.common.collect.Lists; import io.netty.buffer.DrillBuf; import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.memory.BufferAllocator; @@ -60,6 +62,20 @@ public void addField(ValueVector vector) { fieldVectorMap.put(vector.getField(), vector); } + private void replace(ValueVector newVector, SchemaPath schemaPath) { + List vectors = Lists.newArrayList(); + for (VectorWrapper w : container) { + ValueVector vector = w.getValueVector(); + if (vector.getField().getPath().equals(schemaPath)) { + vectors.add(newVector); + } else { + vectors.add(w.getValueVector()); + } + container.remove(vector); + } + container.addCollection(vectors); + } + public Iterator> iterator() { return container.iterator(); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java index 521a41d468d..7f4684175f1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java @@ -19,11 +19,17 @@ package org.apache.drill.exec.vector.complex.writer; import org.apache.drill.BaseTestQuery; +import org.junit.BeforeClass; import org.junit.Test; public class TestComplexTypeReader extends BaseTestQuery{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class); + @BeforeClass + public static void init() throws Exception { + testNoResult("alter session set `exec.enable_union_type` = true"); + } + @Test // Repeated map (map) -> json. public void testX() throws Exception{ diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index 0ae8945c8aa..d549a191629 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -482,4 +482,22 @@ public void testSumWithTypeCase() throws Exception { } } + @Test + public void testUnionExpressionMaterialization() throws Exception { + String query = "select a + b c from cp.`jsoninput/union/b.json`"; + try { + testBuilder() + .sqlQuery(query) + .ordered() + .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") + .baselineColumns("c") + .baselineValues(3L) + .baselineValues(7.0) + .baselineValues(11.0) + .go(); + } finally { + testNoResult("alter session set `exec.enable_union_type` = false"); + } + } + } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java new file mode 100644 index 00000000000..60a2268b4cb --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestPromotableWriter.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex.writer; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.memory.RootAllocatorFactory; +import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.physical.impl.OutputMutator; +import org.apache.drill.exec.store.TestOutputMutator; +import org.apache.drill.exec.util.BatchPrinter; +import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; +import org.junit.Test; + +public class TestPromotableWriter { + + @Test + public void list() throws Exception { + BufferAllocator allocator = RootAllocatorFactory.newRoot(DrillConfig.create()); + TestOutputMutator output = new TestOutputMutator(allocator); + ComplexWriter rootWriter = new VectorContainerWriter(output, true); + MapWriter writer = rootWriter.rootAsMap(); + + + rootWriter.setPosition(0); + { + writer.map("map").bigInt("a").writeBigInt(1); + } + rootWriter.setPosition(1); + { + writer.map("map").float4("a").writeFloat4(2.0f); + } + rootWriter.setPosition(2); + { + writer.map("map").list("a").startList(); + writer.map("map").list("a").endList(); + } + rootWriter.setPosition(3); + { + writer.map("map").list("a").startList(); + writer.map("map").list("a").bigInt().writeBigInt(3); + writer.map("map").list("a").float4().writeFloat4(4); + writer.map("map").list("a").endList(); + } + rootWriter.setValueCount(4); + BatchPrinter.printBatch(output.getContainer()); + } + +} diff --git a/exec/java-exec/src/test/resources/jsoninput/union/b.json b/exec/java-exec/src/test/resources/jsoninput/union/b.json new file mode 100644 index 00000000000..28a6c5c097c --- /dev/null +++ b/exec/java-exec/src/test/resources/jsoninput/union/b.json @@ -0,0 +1,3 @@ +{ a : 1, b : "2" } +{ a : 3.0, b : "4"} +{ a : 5.0, b : 6 } From f11c9dbc9df0b3b230494d4b4e4b245e0bf1fe93 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Wed, 21 Oct 2015 17:00:44 -0700 Subject: [PATCH 4/5] DRILL-3229: Miscellaneous Union-type fixes --- .../org/apache/drill/common/types/Types.java | 4 +- .../templates/EventBasedRecordWriter.java | 6 +- .../codegen/templates/HolderReaderImpl.java | 1 + .../codegen/templates/UnionFunctions.java | 31 +++- .../codegen/templates/UnionListWriter.java | 36 ++--- .../main/codegen/templates/UnionReader.java | 10 +- .../main/codegen/templates/UnionVector.java | 11 +- .../main/codegen/templates/UnionWriter.java | 58 ++++---- .../org/apache/drill/exec/ExecConstants.java | 2 +- .../drill/exec/expr/EvaluationVisitor.java | 4 +- .../exec/expr/ExpressionTreeMaterializer.java | 67 +++++++-- .../drill/exec/expr/fn/ExceptionFunction.java | 53 +++++++ .../exec/expr/fn/impl/UnionFunctions.java | 134 ++++++++++++------ .../impl/filter/FilterRecordBatch.java | 1 + .../impl/project/ProjectRecordBatch.java | 4 + .../impl/project/ProjectorTemplate.java | 1 + .../drill/exec/vector/complex/ListVector.java | 8 ++ .../exec/vector/complex/fn/JsonWriter.java | 2 + .../vector/complex/writer/TestJsonReader.java | 10 +- 19 files changed, 317 insertions(+), 126 deletions(-) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java index 0c468dcb6b8..5e810223a86 100644 --- a/common/src/main/java/org/apache/drill/common/types/Types.java +++ b/common/src/main/java/org/apache/drill/common/types/Types.java @@ -96,7 +96,7 @@ public static boolean isNumericType(final MajorType type) { * {@code INFORMATION_SCHEMA.COLUMNS.TYPE_NAME} would list) */ public static String getSqlTypeName(final MajorType type) { - if (type.getMode() == DataMode.REPEATED) { + if (type.getMode() == DataMode.REPEATED || type.getMinorType() == MinorType.LIST) { return "ARRAY"; } @@ -169,7 +169,7 @@ public static String getSqlTypeName(final MajorType type) { * Gets JDBC type code for given Drill RPC-/protobuf-level type. */ public static int getJdbcTypeCode(final MajorType type) { - if (type.getMode() == DataMode.REPEATED) { + if (type.getMode() == DataMode.REPEATED || type.getMinorType() == MinorType.LIST) { return java.sql.Types.ARRAY; } diff --git a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java index bf447c9c91a..dfe9d043519 100644 --- a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java @@ -133,10 +133,10 @@ public static FieldConverter getConverter(RecordWriter recordWriter, int fieldId } case LIST: - switch (reader.getType().getMode()) { - case REPEATED: +// switch (reader.getType().getMode()) { +// case REPEATED: return recordWriter.getNewRepeatedListConverter(fieldId, fieldName, reader); - } +// } <#list vv.types as type> <#list type.minor as minor> diff --git a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java index 9bb4f82dccd..d0f5c4a0926 100644 --- a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java +++ b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java @@ -130,6 +130,7 @@ public void read(Nullable${name}Holder h) { <#list fields as field> h.${field.name} = holder.${field.name}; + h.isSet = isSet() ? 1 : 0; } diff --git a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java index 8293d8a6892..5a49b93eb10 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java +++ b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java @@ -17,12 +17,12 @@ */ <@pp.dropOutputFile /> -<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionFunctions.java" /> +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/GUnionFunctions.java" /> <#include "/@includes/license.ftl" /> -package org.apache.drill.exec.vector.complex.impl; +package org.apache.drill.exec.expr.fn.impl; <#include "/@includes/vv_imports.ftl" /> import org.apache.drill.exec.expr.DrillSimpleFunc; @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") -public class UnionFunctions { +public class GUnionFunctions { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> @@ -49,7 +49,7 @@ public class UnionFunctions { <#if !minor.class?starts_with("Decimal")> @SuppressWarnings("unused") - @FunctionTemplate(name = "is${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "is_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class UnionIs${name} implements DrillSimpleFunc { @Param UnionHolder in; @@ -58,12 +58,16 @@ public static class UnionIs${name} implements DrillSimpleFunc { public void setup() {} public void eval() { - out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.${name?upper_case} ? 1 : 0; + if (in.isSet == 1) { + out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.${name?upper_case} ? 1 : 0; + } else { + out.value = 0; + } } } @SuppressWarnings("unused") - @FunctionTemplate(name = "as${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "assert_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class CastUnion${name} implements DrillSimpleFunc { @Param UnionHolder in; @@ -84,6 +88,21 @@ public void eval() { @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class Cast${name}ToUnion implements DrillSimpleFunc { + @Param ${name}Holder in; + @Output UnionHolder out; + + public void setup() {} + + public void eval() { + out.reader = new org.apache.drill.exec.vector.complex.impl.${name}HolderReaderImpl(in); + out.isSet = 1; + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class CastNullable${name}ToUnion implements DrillSimpleFunc { + @Param Nullable${name}Holder in; @Output UnionHolder out; diff --git a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java index 35c0bc1bc02..c676769f363 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java +++ b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java @@ -16,6 +16,8 @@ * limitations under the License. */ +import java.lang.UnsupportedOperationException; + <@pp.dropOutputFile /> <@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionListWriter.java" /> @@ -33,9 +35,8 @@ @SuppressWarnings("unused") public class UnionListWriter extends AbstractFieldWriter { - ListVector vector; -// UnionVector data; - UInt4Vector offsets; + private ListVector vector; + private UInt4Vector offsets; private PromotableWriter writer; private boolean inMap = false; private String mapName; @@ -44,7 +45,6 @@ public class UnionListWriter extends AbstractFieldWriter { public UnionListWriter(ListVector vector) { super(null); this.vector = vector; -// this.data = (UnionVector) vector.getDataVector(); this.writer = new PromotableWriter(vector.getDataVector(), vector); this.offsets = vector.getOffsetVector(); } @@ -93,7 +93,11 @@ public void close() throws Exception { public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}(String name) { assert inMap; mapName = name; - return this; + final int nextOffset = offsets.getAccessor().get(idx() + 1); + vector.getMutator().setNotNull(idx()); + writer.setPosition(nextOffset); + ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}(name); + return ${uncappedName}Writer; } @@ -119,7 +123,6 @@ public ListWriter list() { public ListWriter list(String name) { final int nextOffset = offsets.getAccessor().get(idx() + 1); vector.getMutator().setNotNull(idx()); -// data.getMutator().setType(nextOffset, MinorType.MAP); writer.setPosition(nextOffset); ListWriter listWriter = writer.list(name); return listWriter; @@ -146,7 +149,6 @@ public void start() { assert inMap; final int nextOffset = offsets.getAccessor().get(idx() + 1); vector.getMutator().setNotNull(idx()); -// data.getMutator().setType(nextOffset, MinorType.MAP); offsets.getMutator().setSafe(idx() + 1, nextOffset); writer.setPosition(nextOffset); } @@ -168,20 +170,12 @@ public void end() { @Override public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - if (inMap) { - final int nextOffset = offsets.getAccessor().get(idx() + 1); - vector.getMutator().setNotNull(idx()); -// data.getMutator().setType(nextOffset, MinorType.MAP); - writer.setPosition(nextOffset); - ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}(mapName); - ${uncappedName}Writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); - } else { - final int nextOffset = offsets.getAccessor().get(idx() + 1); - vector.getMutator().setNotNull(idx()); - writer.setPosition(nextOffset); - writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); - offsets.getMutator().setSafe(idx() + 1, nextOffset + 1); - } + assert !inMap; + final int nextOffset = offsets.getAccessor().get(idx() + 1); + vector.getMutator().setNotNull(idx()); + writer.setPosition(nextOffset); + writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); + offsets.getMutator().setSafe(idx() + 1, nextOffset + 1); } diff --git a/exec/java-exec/src/main/codegen/templates/UnionReader.java b/exec/java-exec/src/main/codegen/templates/UnionReader.java index 4943ea0debc..7962b675553 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionReader.java +++ b/exec/java-exec/src/main/codegen/templates/UnionReader.java @@ -39,8 +39,16 @@ public UnionReader(UnionVector data) { this.data = data; } + private static MajorType[] TYPES = new MajorType[43]; + + static { + for (MinorType minorType : MinorType.values()) { + TYPES[minorType.getNumber()] = Types.optional(minorType); + } + } + public MajorType getType() { - return Types.required(MinorType.valueOf(data.getTypeValue(idx()))); + return TYPES[data.getTypeValue(idx())]; } public boolean isSet(){ diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java index 3e7b3bf4290..8d647373854 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionVector.java +++ b/exec/java-exec/src/main/codegen/templates/UnionVector.java @@ -111,10 +111,12 @@ public ValueVector getSingleVector() { return singleVector; } + private static final MajorType MAP_TYPE = Types.optional(MinorType.MAP); + public MapVector getMap() { if (mapVector == null) { int vectorCount = internalMap.size(); - mapVector = internalMap.addOrGet("map", Types.optional(MinorType.MAP), MapVector.class); + mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class); updateState(mapVector); addSubType(MinorType.MAP); if (internalMap.size() > vectorCount) { @@ -130,11 +132,12 @@ public MapVector getMap() { <#if !minor.class?starts_with("Decimal")> private Nullable${name}Vector ${uncappedName}Vector; + private static final MajorType ${name?upper_case}_TYPE = Types.optional(MinorType.${name?upper_case}); public Nullable${name}Vector get${name}Vector() { if (${uncappedName}Vector == null) { int vectorCount = internalMap.size(); - ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", Types.optional(MinorType.${name?upper_case}), Nullable${name}Vector.class); + ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class); updateState(${uncappedName}Vector); addSubType(MinorType.${name?upper_case}); if (internalMap.size() > vectorCount) { @@ -148,10 +151,12 @@ public MapVector getMap() { + private static final MajorType LIST_TYPE = Types.optional(MinorType.LIST); + public ListVector getList() { if (listVector == null) { int vectorCount = internalMap.size(); - listVector = internalMap.addOrGet("list", Types.optional(MinorType.LIST), ListVector.class); + listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class); updateState(listVector); addSubType(MinorType.LIST); if (internalMap.size() > vectorCount) { diff --git a/exec/java-exec/src/main/codegen/templates/UnionWriter.java b/exec/java-exec/src/main/codegen/templates/UnionWriter.java index 9d779999c17..9a39a40758a 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionWriter.java +++ b/exec/java-exec/src/main/codegen/templates/UnionWriter.java @@ -63,27 +63,27 @@ public void setPosition(int index) { @Override public void start() { data.getMutator().setType(idx(), MinorType.MAP); - getMapWriter(true).start(); + getMapWriter().start(); } @Override public void end() { - getMapWriter(false).end(); + getMapWriter().end(); } @Override public void startList() { - getListWriter(true).startList(); + getListWriter().startList(); data.getMutator().setType(idx(), MinorType.LIST); } @Override public void endList() { - getListWriter(true).endList(); + getListWriter().endList(); } - private MapWriter getMapWriter(boolean create) { - if (create && mapWriter == null) { + private MapWriter getMapWriter() { + if (mapWriter == null) { mapWriter = new SingleMapWriter(data.getMap(), null, true, false); mapWriter.setPosition(idx()); writers.add(mapWriter); @@ -93,11 +93,11 @@ private MapWriter getMapWriter(boolean create) { public MapWriter asMap() { data.getMutator().setType(idx(), MinorType.MAP); - return getMapWriter(true); + return getMapWriter(); } - private ListWriter getListWriter(boolean create) { - if (create && listWriter == null) { + private ListWriter getListWriter() { + if (listWriter == null) { listWriter = new UnionListWriter(data.getList()); listWriter.setPosition(idx()); writers.add(listWriter); @@ -107,7 +107,7 @@ private ListWriter getListWriter(boolean create) { public ListWriter asList() { data.getMutator().setType(idx(), MinorType.LIST); - return getListWriter(true); + return getListWriter(); } <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> @@ -118,8 +118,8 @@ public ListWriter asList() { private ${name}Writer ${name?uncap_first}Writer; - private ${name}Writer get${name}Writer(boolean create) { - if (create && ${uncappedName}Writer == null) { + private ${name}Writer get${name}Writer() { + if (${uncappedName}Writer == null) { ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null); ${uncappedName}Writer.setPosition(idx()); writers.add(${uncappedName}Writer); @@ -129,20 +129,20 @@ public ListWriter asList() { public ${name}Writer as${name}() { data.getMutator().setType(idx(), MinorType.${name?upper_case}); - return get${name}Writer(true); + return get${name}Writer(); } @Override public void write(${name}Holder holder) { data.getMutator().setType(idx(), MinorType.${name?upper_case}); - get${name}Writer(true).setPosition(idx()); - get${name}Writer(true).write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); + get${name}Writer().setPosition(idx()); + get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { data.getMutator().setType(idx(), MinorType.${name?upper_case}); - get${name}Writer(true).setPosition(idx()); - get${name}Writer(true).write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); + get${name}Writer().setPosition(idx()); + get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); } @@ -154,29 +154,29 @@ public void writeNull() { @Override public MapWriter map() { data.getMutator().setType(idx(), MinorType.LIST); - getListWriter(true).setPosition(idx()); - return getListWriter(true).map(); + getListWriter().setPosition(idx()); + return getListWriter().map(); } @Override public ListWriter list() { data.getMutator().setType(idx(), MinorType.LIST); - getListWriter(true).setPosition(idx()); - return getListWriter(true).list(); + getListWriter().setPosition(idx()); + return getListWriter().list(); } @Override public ListWriter list(String name) { data.getMutator().setType(idx(), MinorType.MAP); - getMapWriter(true).setPosition(idx()); - return getMapWriter(true).list(name); + getMapWriter().setPosition(idx()); + return getMapWriter().list(name); } @Override public MapWriter map(String name) { data.getMutator().setType(idx(), MinorType.MAP); - getMapWriter(true).setPosition(idx()); - return getMapWriter(true).map(name); + getMapWriter().setPosition(idx()); + return getMapWriter().map(name); } <#list vv.types as type><#list type.minor as minor> @@ -188,15 +188,15 @@ public MapWriter map(String name) { @Override public ${capName}Writer ${lowerName}(String name) { data.getMutator().setType(idx(), MinorType.MAP); - getMapWriter(true).setPosition(idx()); - return getMapWriter(true).${lowerName}(name); + getMapWriter().setPosition(idx()); + return getMapWriter().${lowerName}(name); } @Override public ${capName}Writer ${lowerName}() { data.getMutator().setType(idx(), MinorType.LIST); - getListWriter(true).setPosition(idx()); - return getListWriter(true).${lowerName}(); + getListWriter().setPosition(idx()); + return getListWriter().${lowerName}(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index f85a776b907..afa9458fdb6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -244,7 +244,7 @@ public interface ExecConstants { Long.MAX_VALUE, 60 * 1000 * 5); public static final String ENABLE_VERBOSE_ERRORS_KEY = "exec.errors.verbose"; - public static final OptionValidator ENABLE_VERBOSE_ERRORS = new BooleanValidator(ENABLE_VERBOSE_ERRORS_KEY, false); + public static final OptionValidator ENABLE_VERBOSE_ERRORS = new BooleanValidator(ENABLE_VERBOSE_ERRORS_KEY, true); public static final String ENABLE_NEW_TEXT_READER_KEY = "exec.storage.enable_new_text_reader"; public static final OptionValidator ENABLE_NEW_TEXT_READER = new BooleanValidator(ENABLE_NEW_TEXT_READER_KEY, true); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index 424174ed657..72a400d537e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -413,7 +413,9 @@ private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpressio if (out.isOptional()) { ifNoVal.assign(out.getIsSet(), JExpr.lit(0)); } - ifNoVal.assign(isNull, JExpr.lit(1)); + if (isNull != null) { + ifNoVal.assign(isNull, JExpr.lit(1)); + } ifNoVal._break(label); expr = list.invoke("reader"); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 827623b2094..741fbb90d1a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -19,10 +19,15 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Deque; +import java.util.HashSet; import java.util.List; import java.util.Queue; +import java.util.Set; +import java.util.Stack; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; @@ -74,6 +79,7 @@ import org.apache.drill.exec.expr.fn.AbstractFuncHolder; import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder; import org.apache.drill.exec.expr.fn.DrillFuncHolder; +import org.apache.drill.exec.expr.fn.ExceptionFunction; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.fn.FunctionLookupContext; import org.apache.drill.exec.record.TypedFieldId; @@ -145,6 +151,10 @@ public static LogicalExpression addCastExpression(LogicalExpression fromExpr, Ma List castArgs = Lists.newArrayList(); castArgs.add(fromExpr); //input_expr + if (fromExpr.getMajorType().getMinorType() == MinorType.UNION && toType.getMinorType() == MinorType.UNION) { + return fromExpr; + } + if (!Types.isFixedWidthType(toType) && !Types.isUnion(toType)) { /* We are implicitly casting to VARCHAR so we don't have a max length, @@ -196,7 +206,8 @@ private static void logFunctionResolutionError(ErrorCollector errorCollector, Fu private static class MaterializeVisitor extends AbstractExprVisitor { private ExpressionValidator validator = new ExpressionValidator(); - private final ErrorCollector errorCollector; + private ErrorCollector errorCollector; + private Deque errorCollectors = new ArrayDeque<>(); private final VectorAccessible batch; private final boolean allowComplexWriter; private final boolean unionTypeEnabled; @@ -250,9 +261,9 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionLookupCont //replace with a new function call, since its argument could be changed. call = new FunctionCall(call.getName(), args, call.getPosition()); - if (hasUnionInput(call)) { - return rewriteUnionFunction(call, functionLookupContext); - } +// if (hasUnionInput(call)) { +// return rewriteUnionFunction(call, functionLookupContext); +// } FunctionResolver resolver = FunctionResolverFactory.getResolver(call); DrillFuncHolder matchedFuncHolder = functionLookupContext.findDrillFunction(resolver, call); @@ -323,16 +334,29 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionLookupCont return matchedNonDrillFuncHolder.getExpr(call.getName(), extArgsWithCast, call.getPosition()); } + if (hasUnionInput(call)) { + return rewriteUnionFunction(call, functionLookupContext); + } + logFunctionResolutionError(errorCollector, call); return NullExpression.INSTANCE; } + private static final Set UNION_FUNCTIONS; + + static { + UNION_FUNCTIONS = new HashSet<>(); + for (MinorType t : MinorType.values()) { + UNION_FUNCTIONS.add("assert_" + t.name().toLowerCase()); + UNION_FUNCTIONS.add("is_" + t.name().toLowerCase()); + } + UNION_FUNCTIONS.add("typeof"); + } + private boolean hasUnionInput(FunctionCall call) { for (LogicalExpression arg : call.args) { if (arg.getMajorType().getMinorType() == MinorType.UNION) { - if (!call.getName().toLowerCase().startsWith("as") && !call.getName().toLowerCase().startsWith("is") && !call.getName().toLowerCase().startsWith("typeof")) { - return true; - } + return true; } } return false; @@ -364,7 +388,17 @@ private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookup newArgs.add(e.accept(new CloneVisitor(), null)); } - LogicalExpression thenExpression = new FunctionCall(call.getName(), newArgs, call.getPosition()); + errorCollectors.push(errorCollector); + errorCollector = new ErrorCollectorImpl(); + + LogicalExpression thenExpression = new FunctionCall(call.getName(), newArgs, call.getPosition()).accept(this, functionLookupContext); + + if (errorCollector.hasErrors()) { + thenExpression = getExceptionFunction(errorCollector.toErrorString()); + } + + errorCollector = errorCollectors.pop(); + IfExpression.IfCondition condition = new IfCondition(ifCondition, thenExpression); ifConditions.add(condition); } @@ -381,18 +415,29 @@ private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookup throw new UnsupportedOperationException("Did not find any Union input types"); } + private LogicalExpression getExceptionFunction(String message) { + QuotedString msg = new QuotedString(message, ExpressionPosition.UNKNOWN); + List args = Lists.newArrayList(); + args.add(msg); + FunctionCall call = new FunctionCall(ExceptionFunction.EXCEPTION_FUNCTION_NAME, args, ExpressionPosition.UNKNOWN); + return call; + } + private LogicalExpression getUnionCastExpressionForType(MinorType type, LogicalExpression arg) { if (type == MinorType.UNION) { - return null; + return arg; + } + if (type == MinorType.LIST || type == MinorType.MAP) { + return getExceptionFunction("Unable to cast union to " + type); } - String castFuncName = String.format("as%s", type.toString()); + String castFuncName = String.format("assert_%s", type.toString()); List args = Lists.newArrayList(); args.add(arg); return new FunctionCall(castFuncName, args, ExpressionPosition.UNKNOWN); } private LogicalExpression getIsTypeExpressionForType(MinorType type, LogicalExpression arg) { - String isFuncName = String.format("is%s", type.toString()); + String isFuncName = String.format("is_%s", type.toString()); List args = Lists.newArrayList(); args.add(arg); return new FunctionCall(isFuncName, args, ExpressionPosition.UNKNOWN); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java new file mode 100644 index 00000000000..04b75baa3bb --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn; + +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.BigIntHolder; +import org.apache.drill.exec.expr.holders.VarCharHolder; + +public class ExceptionFunction { + + public static final String EXCEPTION_FUNCTION_NAME = "throwException"; + + @FunctionTemplate(name = EXCEPTION_FUNCTION_NAME, isRandom = true, + scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) + public static class ThrowException implements DrillSimpleFunc { + + @Param VarCharHolder message; + @Output BigIntHolder out; + + public void setup() { + } + + public void eval() { + byte[] bytes = new byte[message.end - message.start]; + message.buffer.getBytes(message.start, bytes, 0, message.end - message.start); + org.apache.drill.exec.expr.fn.ExceptionFunction.throwException(new String(bytes)); + } + } + + public static void throwException(String message) { + throw new org.apache.drill.common.exceptions.DrillRuntimeException(message); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java index 34b14fed929..9e9a6e3020a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java @@ -18,11 +18,14 @@ package org.apache.drill.exec.expr.fn.impl; import io.netty.buffer.DrillBuf; +import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.expr.DrillSimpleFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.expr.annotations.Output; import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.BigIntHolder; +import org.apache.drill.exec.expr.holders.BitHolder; import org.apache.drill.exec.expr.holders.NullableIntHolder; import org.apache.drill.exec.expr.holders.NullableUInt1Holder; import org.apache.drill.exec.expr.holders.UnionHolder; @@ -35,90 +38,91 @@ public class UnionFunctions { - @FunctionTemplate(names = {"typeString"}, + @FunctionTemplate(names = {"typeOf"}, scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = NullHandling.NULL_IF_NULL) - public static class FromType implements DrillSimpleFunc { + nulls = NullHandling.INTERNAL) + public static class GetType implements DrillSimpleFunc { @Param - IntHolder in; + FieldReader input; @Output VarCharHolder out; @Inject - DrillBuf buffer; + DrillBuf buf; public void setup() {} public void eval() { - VarCharHolder h = org.apache.drill.exec.vector.ValueHolderHelper.getVarCharHolder(buffer, org.apache.drill.common.types.MinorType.valueOf(in.value).toString()); - out.buffer = h.buffer; - out.start = h.start; - out.end = h.end; + byte[] type; + if (input.isSet()) { + type = input.getType().getMinorType().name().getBytes(); + } else { + type = org.apache.drill.common.types.TypeProtos.MinorType.NULL.name().getBytes(); + } + buf = buf.reallocIfNeeded(type.length); + buf.setBytes(0, type); + out.buffer = buf; + out.start = 0; + out.end = type.length; } } - @FunctionTemplate(names = {"type"}, - scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = NullHandling.NULL_IF_NULL) - public static class ToType implements DrillSimpleFunc { + @SuppressWarnings("unused") + @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class CastUnionToUnion implements DrillSimpleFunc{ - @Param - VarCharHolder input; + @Param FieldReader in; @Output - IntHolder out; + UnionHolder out; public void setup() {} public void eval() { - - out.value = input.getType().getMinorType().getNumber(); - byte[] b = new byte[input.end - input.start]; - input.buffer.getBytes(input.start, b, 0, b.length); - String type = new String(b); - out.value = org.apache.drill.common.types.MinorType.valueOf(type.toUpperCase()).getNumber(); + out.reader = in; + out.isSet = in.isSet() ? 1 : 0; } } - @FunctionTemplate(names = {"typeOf"}, - scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = NullHandling.INTERNAL) - public static class GetType implements DrillSimpleFunc { + @SuppressWarnings("unused") + @FunctionTemplate(name = "asList", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class CastUnionList implements DrillSimpleFunc { - @Param - FieldReader input; - @Output - NullableIntHolder out; + @Param UnionHolder in; + @Output UnionHolder out; public void setup() {} public void eval() { - - out.value = input.isSet() ? input.getType().getMinorType().getNumber() : 0; - out.isSet = 1; - + if (in.isSet == 1) { + out.reader = in.reader; + } else { + out.isSet = 0; + } } } @SuppressWarnings("unused") - @FunctionTemplate(names = {"castUNION", "castToUnion"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) - public static class CastUnionToUnion implements DrillSimpleFunc{ + @FunctionTemplate(name = "is_LIST", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class UnionIsList implements DrillSimpleFunc { - @Param FieldReader in; - @Output - UnionHolder out; + @Param UnionHolder in; + @Output BitHolder out; public void setup() {} public void eval() { - out.reader = in; - out.isSet = in.isSet() ? 1 : 0; + if (in.isSet == 1) { + out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.LIST ? 1 : 0; + } else { + out.value = 0; + } } } @SuppressWarnings("unused") - @FunctionTemplate(name = "asList", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) - public static class CastUnionList implements DrillSimpleFunc { + @FunctionTemplate(name = "asMap", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class CastUnionMap implements DrillSimpleFunc { @Param UnionHolder in; @Output UnionHolder out; @@ -133,4 +137,48 @@ public void eval() { } } } + + @SuppressWarnings("unused") + @FunctionTemplate(name = "is_MAP", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + public static class UnionIsMap implements DrillSimpleFunc { + + @Param UnionHolder in; + @Output BitHolder out; + + public void setup() {} + + public void eval() { + if (in.isSet == 1) { + out.value = in.getType().getMinorType() == org.apache.drill.common.types.TypeProtos.MinorType.MAP ? 1 : 0; + } else { + out.value = 0; + } + } + } + + @FunctionTemplate(names = {"isnotnull", "is not null"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL) + public static class IsNotNull implements DrillSimpleFunc { + + @Param UnionHolder input; + @Output BitHolder out; + + public void setup() { } + + public void eval() { + out.value = input.isSet == 1 ? 1 : 0; + } + } + + @FunctionTemplate(names = {"isnull", "is null"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL) + public static class IsNull implements DrillSimpleFunc { + + @Param UnionHolder input; + @Output BitHolder out; + + public void setup() { } + + public void eval() { + out.value = input.isSet == 1 ? 0 : 1; + } + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java index fd16bc84153..0a96285096e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java @@ -22,6 +22,7 @@ import org.apache.drill.common.expression.ErrorCollector; import org.apache.drill.common.expression.ErrorCollectorImpl; +import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.ClassTransformationException; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index 2bffded612d..2e3a403451c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -104,6 +104,10 @@ private void clear() { } } + public static void crap() { + System.out.print(""); + } + public ProjectRecordBatch(final Project pop, final RecordBatch incoming, final FragmentContext context) throws OutOfMemoryException { super(pop, context, incoming); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java index a6294d89d05..ff04e0d7d48 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java @@ -59,6 +59,7 @@ public final int projectRecords(int startIndex, final int recordCount, int first final int countN = recordCount; int i; for (i = startIndex; i < startIndex + countN; i++, firstOutputIndex++) { + ProjectRecordBatch.crap(); doEval(i, firstOutputIndex); } if (i < startIndex + recordCount || startIndex > 0) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java index a44efe343fe..9a0f4b754cc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java @@ -39,6 +39,7 @@ import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VectorDescriptor; +import org.apache.drill.exec.vector.ZeroVector; import org.apache.drill.exec.vector.complex.impl.ComplexCopier; import org.apache.drill.exec.vector.complex.impl.UnionListReader; import org.apache.drill.exec.vector.complex.impl.UnionListWriter; @@ -80,9 +81,16 @@ public void allocateNew() throws OutOfMemoryRuntimeException { public void transferTo(ListVector target) { offsets.makeTransferPair(target.offsets).transfer(); bits.makeTransferPair(target.bits).transfer(); + if (target.getDataVector() instanceof ZeroVector) { + target.addOrGetVector(new VectorDescriptor(vector.getField().getType())); + } getDataVector().makeTransferPair(target.getDataVector()).transfer(); } + public void copyFromSafe(int inIndex, int outIndex, ListVector from) { + copyFrom(inIndex, outIndex, from); + } + public void copyFrom(int inIndex, int outIndex, ListVector from) { FieldReader in = from.getReader(); in.setPosition(inIndex); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java index 6ff7d905f90..fe43f82304d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java @@ -27,6 +27,8 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerator; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; public class JsonWriter { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonWriter.class); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index d549a191629..e03fb21a317 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -433,14 +433,14 @@ public void testSelectStarWithUnionType() throws Exception { @Test public void testSelectFromListWithCase() throws Exception { - String query = "select a from (select case when typeOf(field2) = type('list') then asBigInt(field2[4][1].inner7) end a from cp.`jsoninput/union/a.json`) where a is not null"; + String query = "select a, typeOf(a) `type` from (select case when is_list(field2) then field2[4][1].inner7 end a from cp.`jsoninput/union/a.json`) where a is not null"; try { testBuilder() .sqlQuery(query) .ordered() .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true") - .baselineColumns("a") - .baselineValues(13L) + .baselineColumns("a", "type") + .baselineValues(13L, "BIGINT") .go(); } finally { testNoResult("alter session set `exec.enable_union_type` = false"); @@ -449,7 +449,7 @@ public void testSelectFromListWithCase() throws Exception { @Test public void testTypeCase() throws Exception { - String query = "select case typeOf(field1) when type('bigint') then asBigInt(field1) when type('list') then asBigInt(field1[0]) when type('map') then asBigInt(t.field1.inner1) end f1 from cp.`jsoninput/union/a.json` t"; + String query = "select case when is_bigint(field1) then field1 when is_list(field1) then field1[0] when is_map(field1) then t.field1.inner1 end f1 from cp.`jsoninput/union/a.json` t"; try { testBuilder() .sqlQuery(query) @@ -468,7 +468,7 @@ public void testTypeCase() throws Exception { @Test public void testSumWithTypeCase() throws Exception { - String query = "select sum(f1) sum_f1 from (select case typeOf(field1) when type('bigint') then asBigInt(field1) when type('list') then asBigInt(field1[0]) when type('map') then asBigInt(t.field1.inner1) end f1 from cp.`jsoninput/union/a.json` t)"; + String query = "select sum(cast(f1 as bigint)) sum_f1 from (select case when is_bigint(field1) then field1 when is_list(field1) then field1[0] when is_map(field1) then t.field1.inner1 end f1 from cp.`jsoninput/union/a.json` t)"; try { testBuilder() .sqlQuery(query) From 93bacd49b079c95e8eef0ee2f838c2710a3703a6 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Fri, 30 Oct 2015 00:30:22 -0700 Subject: [PATCH 5/5] Address review comments Also fix bug when doing a flatten on a List vector --- .../org/apache/drill/common/types/Types.java | 5 +- .../templates/AbstractFieldReader.java | 8 +- .../main/codegen/templates/ComplexCopier.java | 23 ++-- .../templates/EventBasedRecordWriter.java | 10 +- .../main/codegen/templates/MapWriters.java | 13 +- .../main/codegen/templates/NullReader.java | 2 +- .../codegen/templates/UnionFunctions.java | 7 +- .../main/codegen/templates/UnionReader.java | 3 +- .../main/codegen/templates/UnionVector.java | 79 +++++------- .../main/codegen/templates/UnionWriter.java | 2 +- .../apache/drill/exec/expr/CloneVisitor.java | 3 + .../drill/exec/expr/EvaluationVisitor.java | 10 +- .../exec/expr/ExpressionTreeMaterializer.java | 62 ++++++--- .../drill/exec/expr/fn/ExceptionFunction.java | 8 +- .../exec/expr/fn/impl/UnionFunctions.java | 17 ++- .../drill/exec/physical/impl/ImplCreator.java | 1 + .../physical/impl/aggregate/HashAggBatch.java | 2 +- .../impl/aggregate/StreamingAggBatch.java | 2 +- .../impl/flatten/FlattenRecordBatch.java | 4 +- .../impl/project/ProjectRecordBatch.java | 4 - .../impl/project/ProjectorTemplate.java | 1 - .../IteratorValidatorBatchIterator.java | 1 - .../exec/record/SimpleVectorWrapper.java | 2 +- .../vector/accessor/UnionSqlAccessor.java | 2 +- .../complex/AbstractContainerVector.java | 78 +---------- .../vector/complex/AbstractMapVector.java | 2 +- .../exec/vector/complex/FieldIdUtil.java | 122 ++++++++++++++++++ .../drill/exec/vector/complex/ListVector.java | 83 ++---------- .../complex/impl/AbstractBaseReader.java | 3 +- .../complex/impl/ComplexWriterImpl.java | 4 +- .../vector/complex/impl/PromotableWriter.java | 5 +- .../vector/complex/impl/UnionListReader.java | 3 +- .../complex/impl/VectorContainerWriter.java | 2 +- .../org/apache/drill/TestExampleQueries.java | 9 -- 34 files changed, 281 insertions(+), 301 deletions(-) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java index 5e810223a86..74b313ecd2d 100644 --- a/common/src/main/java/org/apache/drill/common/types/Types.java +++ b/common/src/main/java/org/apache/drill/common/types/Types.java @@ -35,10 +35,7 @@ public class Types { public static final MajorType OPTIONAL_BIT = optional(MinorType.BIT); public static boolean isUnion(MajorType toType) { - if (toType.getMinorType() == MinorType.UNION) { - return true; - } - return false; + return toType.getMinorType() == MinorType.UNION; } public static enum Comparability { diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java index a4d22cf08a3..2b7b305038a 100644 --- a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java +++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java @@ -33,6 +33,10 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe super(); } + /** + * Returns true if the current value of the reader is not null + * @return + */ public boolean isSet() { return true; } @@ -61,9 +65,7 @@ public void copyAsValue(MapWriter writer){ public void copyAsField(String name, MapWriter writer){ fail("CopyAsField MapWriter"); } -// public void copyAsValue(ListWriter writer){ -// fail("CopyAsValueList"); -// } + public void copyAsField(String name, ListWriter writer){ fail("CopyAsFieldList"); } diff --git a/exec/java-exec/src/main/codegen/templates/ComplexCopier.java b/exec/java-exec/src/main/codegen/templates/ComplexCopier.java index c3b5cb56b0e..b8b76161fb8 100644 --- a/exec/java-exec/src/main/codegen/templates/ComplexCopier.java +++ b/exec/java-exec/src/main/codegen/templates/ComplexCopier.java @@ -32,19 +32,16 @@ @SuppressWarnings("unused") public class ComplexCopier { - FieldReader in; - FieldWriter out; - - public ComplexCopier(FieldReader in, FieldWriter out) { - this.in = in; - this.out = out; - } - - public void write() { - writeValue(in, out); + /** + * Do a deep copy of the value in input into output + * @param in + * @param out + */ + public static void copy(FieldReader input, FieldWriter output) { + writeValue(input, output); } - private void writeValue(FieldReader reader, FieldWriter writer) { + private static void writeValue(FieldReader reader, FieldWriter writer) { final DataMode m = reader.getType().getMode(); final MinorType mt = reader.getType().getMinorType(); @@ -94,7 +91,7 @@ private void writeValue(FieldReader reader, FieldWriter writer) { } } - private FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) { + private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) { switch (reader.getType().getMinorType()) { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> @@ -113,7 +110,7 @@ private FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, } } - private FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) { + private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) { switch (reader.getType().getMinorType()) { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> diff --git a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java index dfe9d043519..c7676cbb6a6 100644 --- a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java @@ -119,10 +119,9 @@ public void endField() throws IOException { } public static FieldConverter getConverter(RecordWriter recordWriter, int fieldId, String fieldName, FieldReader reader) { - if (reader instanceof UnionReader) { - return recordWriter.getNewUnionConverter(fieldId, fieldName, reader); - } switch (reader.getType().getMinorType()) { + case UNION: + return recordWriter.getNewUnionConverter(fieldId, fieldName, reader); case MAP: switch (reader.getType().getMode()) { case REQUIRED: @@ -133,10 +132,7 @@ public static FieldConverter getConverter(RecordWriter recordWriter, int fieldId } case LIST: -// switch (reader.getType().getMode()) { -// case REPEATED: - return recordWriter.getNewRepeatedListConverter(fieldId, fieldName, reader); -// } + return recordWriter.getNewRepeatedListConverter(fieldId, fieldName, reader); <#list vv.types as type> <#list type.minor as minor> diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java index e6becf2a950..9a85ebf8b15 100644 --- a/exec/java-exec/src/main/codegen/templates/MapWriters.java +++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java @@ -53,17 +53,15 @@ public class ${mode}MapWriter extends AbstractFieldWriter { <#if mode == "Repeated">private int currentChildIndex = 0; private final boolean unionEnabled; - private final boolean unionInternalMap; - public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled, boolean unionInternalMap) { + public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled) { super(parent); this.container = container; this.unionEnabled = unionEnabled; - this.unionInternalMap = unionInternalMap; } public ${mode}MapWriter(${containerClass} container, FieldWriter parent) { - this(container, parent, false, false); + this(container, parent, false); } @Override @@ -81,11 +79,10 @@ public MapWriter map(String name) { FieldWriter writer = fields.get(name.toLowerCase()); if(writer == null){ int vectorCount=container.size(); - if(!unionEnabled || unionInternalMap){ - MapVector vector=container.addOrGet(name,MapVector.TYPE,MapVector.class); - writer=new SingleMapWriter(vector,this); - } else { MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class); + if(!unionEnabled){ + writer = new SingleMapWriter(vector, this); + } else { writer = new PromotableWriter(vector, container); } if(vectorCount != container.size()) { diff --git a/exec/java-exec/src/main/codegen/templates/NullReader.java b/exec/java-exec/src/main/codegen/templates/NullReader.java index 472dbed557c..62aa33ec6e6 100644 --- a/exec/java-exec/src/main/codegen/templates/NullReader.java +++ b/exec/java-exec/src/main/codegen/templates/NullReader.java @@ -60,7 +60,7 @@ public void copyAsValue(UnionWriter writer) {} <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> public void read(${name}Holder holder){ - throw new UnsupportedOperationException("NullReader cannot read into non-nullable holder"); + throw new UnsupportedOperationException("NullReader cannot write into non-nullable holder"); } public void read(Nullable${name}Holder holder){ diff --git a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java index 5a49b93eb10..be7901feaf8 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionFunctions.java +++ b/exec/java-exec/src/main/codegen/templates/UnionFunctions.java @@ -40,6 +40,9 @@ */ @SuppressWarnings("unused") +/** + * Additional functions can be found in the class UnionFunctions + */ public class GUnionFunctions { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> @@ -49,7 +52,7 @@ public class GUnionFunctions { <#if !minor.class?starts_with("Decimal")> @SuppressWarnings("unused") - @FunctionTemplate(name = "is_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "IS_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class UnionIs${name} implements DrillSimpleFunc { @Param UnionHolder in; @@ -67,7 +70,7 @@ public void eval() { } @SuppressWarnings("unused") - @FunctionTemplate(name = "assert_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "ASSERT_${name?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class CastUnion${name} implements DrillSimpleFunc { @Param UnionHolder in; diff --git a/exec/java-exec/src/main/codegen/templates/UnionReader.java b/exec/java-exec/src/main/codegen/templates/UnionReader.java index 7962b675553..46a32ee9b44 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionReader.java +++ b/exec/java-exec/src/main/codegen/templates/UnionReader.java @@ -166,8 +166,7 @@ public void copyAsValue(${name}Writer writer){ @Override public void copyAsValue(ListWriter writer) { - ComplexCopier copier = new ComplexCopier(this, (FieldWriter) writer); - copier.write(); + ComplexCopier.copy(this, (FieldWriter) writer); } @Override diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java index 8d647373854..1b64cdd39fa 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionVector.java +++ b/exec/java-exec/src/main/codegen/templates/UnionVector.java @@ -19,12 +19,12 @@ import org.apache.drill.common.types.TypeProtos.MinorType; <@pp.dropOutputFile /> -<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionVector.java" /> +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/UnionVector.java" /> <#include "/@includes/license.ftl" /> -package org.apache.drill.exec.vector.complex.impl; +package org.apache.drill.exec.vector.complex; <#include "/@includes/vv_imports.ftl" /> import java.util.Iterator; @@ -37,6 +37,14 @@ @SuppressWarnings("unused") +/** + * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each + * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, + * as well as the addOrGet method. + * + * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup + * each time the vector is accessed. + */ public class UnionVector implements ValueVector { private MaterializedField field; @@ -46,50 +54,30 @@ public class UnionVector implements ValueVector { private int valueCount; private MapVector internalMap; - private SingleMapWriter internalMapWriter; private UInt1Vector typeVector; private MapVector mapVector; private ListVector listVector; - private NullableBigIntVector bigInt; - private NullableVarCharVector varChar; private FieldReader reader; private NullableBitVector bit; - private State state = State.INIT; private int singleType = 0; private ValueVector singleVector; private MajorType majorType; private final CallBack callBack; - private enum State { - INIT, SINGLE, MULTI - } - public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) { this.field = field.clone(); this.allocator = allocator; - internalMap = new MapVector("internal", allocator, callBack); - internalMapWriter = new SingleMapWriter(internalMap, null, true, true); + this.internalMap = new MapVector("internal", allocator, callBack); this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class); this.field.addChild(internalMap.getField().clone()); this.majorType = field.getType(); this.callBack = callBack; } - private void updateState(ValueVector v) { - if (state == State.INIT) { - state = State.SINGLE; - singleVector = v; - singleType = v.getField().getType().getMinorType().getNumber(); - } else { - state = State.MULTI; - singleVector = null; - } - } - public List getSubTypes() { return majorType.getSubTypeList(); } @@ -101,23 +89,12 @@ private void addSubType(MinorType type) { } } - public boolean isSingleType() { - return state == State.SINGLE && singleType != MinorType.LIST_VALUE; - } - - public ValueVector getSingleVector() { - assert state != State.MULTI : "Cannot get single vector when there are multiple types"; - assert state != State.INIT : "Cannot get single vector when there are no types"; - return singleVector; - } - private static final MajorType MAP_TYPE = Types.optional(MinorType.MAP); public MapVector getMap() { if (mapVector == null) { int vectorCount = internalMap.size(); mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class); - updateState(mapVector); addSubType(MinorType.MAP); if (internalMap.size() > vectorCount) { mapVector.allocateNew(); @@ -138,7 +115,6 @@ public MapVector getMap() { if (${uncappedName}Vector == null) { int vectorCount = internalMap.size(); ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class); - updateState(${uncappedName}Vector); addSubType(MinorType.${name?upper_case}); if (internalMap.size() > vectorCount) { ${uncappedName}Vector.allocateNew(); @@ -157,7 +133,6 @@ public ListVector getList() { if (listVector == null) { int vectorCount = internalMap.size(); listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class); - updateState(listVector); addSubType(MinorType.LIST); if (internalMap.size() > vectorCount) { listVector.allocateNew(); @@ -240,8 +215,7 @@ public void transferTo(UnionVector target) { public void copyFrom(int inIndex, int outIndex, UnionVector from) { from.getReader().setPosition(inIndex); getWriter().setPosition(outIndex); - ComplexCopier copier = new ComplexCopier(from.reader, mutator.writer); - copier.write(); + ComplexCopier.copy(from.reader, mutator.writer); } public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { @@ -249,7 +223,9 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { } public void addVector(ValueVector v) { - internalMap.putChild(v.getField().getType().getMinorType().name().toLowerCase(), v); + String name = v.getField().getType().getMinorType().name().toLowerCase(); + Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name)); + internalMap.putChild(name, v); addSubType(v.getField().getType().getMinorType()); } @@ -427,9 +403,6 @@ public void setValueCount(int valueCount) { internalMap.getMutator().setValueCount(valueCount); } - public void set(int index, byte[] bytes) { - } - public void setSafe(int index, UnionHolder holder) { FieldReader reader = holder.reader; if (writer == null) { @@ -445,20 +418,16 @@ public void setSafe(int index, UnionHolder holder) { case ${name?upper_case}: Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder(); reader.read(${uncappedName}Holder); - if (holder.isSet == 1) { - writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, ); - } + setSafe(index, ${uncappedName}Holder); break; case MAP: { - ComplexCopier copier = new ComplexCopier(reader, writer); - copier.write(); + ComplexCopier.copy(reader, writer); break; } case LIST: { - ComplexCopier copier = new ComplexCopier(reader, writer); - copier.write(); + ComplexCopier.copy(reader, writer); break; } default: @@ -466,6 +435,18 @@ public void setSafe(int index, UnionHolder holder) { } } + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.class?starts_with("Decimal")> + public void setSafe(int index, Nullable${name}Holder holder) { + setType(index, MinorType.${name?upper_case}); + get${name}Vector().getMutator().setSafe(index, holder); + } + + + + public void setType(int index, MinorType type) { typeVector.getMutator().setSafe(index, type.getNumber()); } diff --git a/exec/java-exec/src/main/codegen/templates/UnionWriter.java b/exec/java-exec/src/main/codegen/templates/UnionWriter.java index 9a39a40758a..7a123b4e799 100644 --- a/exec/java-exec/src/main/codegen/templates/UnionWriter.java +++ b/exec/java-exec/src/main/codegen/templates/UnionWriter.java @@ -84,7 +84,7 @@ public void endList() { private MapWriter getMapWriter() { if (mapWriter == null) { - mapWriter = new SingleMapWriter(data.getMap(), null, true, false); + mapWriter = new SingleMapWriter(data.getMap(), null, true); mapWriter.setPosition(idx()); writers.add(mapWriter); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java index e4fa260324f..b1fb5e768e1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java @@ -49,6 +49,9 @@ import java.util.List; +/** + * Creates a deep copy of a LogicalExpression. Specifically, it creates new instances of the literal expressions + */ public class CloneVisitor extends AbstractExprVisitor { @Override public LogicalExpression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index 72a400d537e..0c65835a6ac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -351,10 +351,10 @@ private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpressio // evaluation work. HoldingContainer out = generator.declare(e.getMajorType()); - final boolean primitive = !Types.usesHolderForGet(e.getMajorType()); final boolean hasReadPath = e.hasReadPath(); final boolean complex = Types.isComplex(e.getMajorType()); final boolean repeated = Types.isRepeated(e.getMajorType()); + final boolean listVector = e.getTypedFieldId().isListVector(); int[] fieldIds = e.getFieldId().getFieldIds(); for (int i = 1; i < fieldIds.length; i++) { @@ -372,7 +372,7 @@ private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpressio PathSegment seg = e.getReadPath(); JVar isNull = null; - boolean isNullReaderLikely = isNullReaderLikely(seg, complex || repeated); + boolean isNullReaderLikely = isNullReaderLikely(seg, complex || repeated || listVector); if (isNullReaderLikely) { isNull = generator.getEvalBlock().decl(generator.getModel().INT, generator.getNextVar("isNull"), JExpr.lit(0)); } @@ -389,7 +389,7 @@ private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpressio if (seg.isArray()) { // stop once we get to the last segment and the final type is neither complex nor repeated (map, list, repeated list). // In case of non-complex and non-repeated type, we return Holder, in stead of FieldReader. - if (seg.isLastPath() && !complex && !repeated && !e.getTypedFieldId().isListVector()) { + if (seg.isLastPath() && !complex && !repeated && !listVector) { break; } @@ -413,9 +413,7 @@ private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpressio if (out.isOptional()) { ifNoVal.assign(out.getIsSet(), JExpr.lit(0)); } - if (isNull != null) { - ifNoVal.assign(isNull, JExpr.lit(1)); - } + ifNoVal.assign(isNull, JExpr.lit(1)); ifNoVal._break(label); expr = list.invoke("reader"); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 741fbb90d1a..715cc55c46f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -17,20 +17,15 @@ */ package org.apache.drill.exec.expr; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.List; import java.util.Queue; import java.util.Set; -import java.util.Stack; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.base.Preconditions; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.BooleanOperator; @@ -44,7 +39,6 @@ import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.LogicalExpressionBase; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.TypedNullConstant; @@ -80,7 +74,6 @@ import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder; import org.apache.drill.exec.expr.fn.DrillFuncHolder; import org.apache.drill.exec.expr.fn.ExceptionFunction; -import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.fn.FunctionLookupContext; import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.record.VectorAccessible; @@ -114,6 +107,11 @@ public static LogicalExpression materializeAndCheckErrors(LogicalExpression expr return e; } + public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext, + boolean allowComplexWriterExpr) { + return materialize(expr, batch, errorCollector, functionLookupContext, allowComplexWriterExpr, false); + } + public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext, boolean allowComplexWriterExpr, boolean unionTypeEnabled) { LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr, unionTypeEnabled), functionLookupContext); @@ -210,6 +208,10 @@ private static class MaterializeVisitor extends AbstractExprVisitor errorCollectors = new ArrayDeque<>(); private final VectorAccessible batch; private final boolean allowComplexWriter; + /** + * If this is false, the materializer will not handle or create UnionTypes + * Once this code is more well tested, we will probably remove this flag + */ private final boolean unionTypeEnabled; public MaterializeVisitor(VectorAccessible batch, ErrorCollector errorCollector, boolean allowComplexWriter, boolean unionTypeEnabled) { @@ -261,10 +263,6 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionLookupCont //replace with a new function call, since its argument could be changed. call = new FunctionCall(call.getName(), args, call.getPosition()); -// if (hasUnionInput(call)) { -// return rewriteUnionFunction(call, functionLookupContext); -// } - FunctionResolver resolver = FunctionResolverFactory.getResolver(call); DrillFuncHolder matchedFuncHolder = functionLookupContext.findDrillFunction(resolver, call); @@ -362,6 +360,14 @@ private boolean hasUnionInput(FunctionCall call) { return false; } + /** + * Converts a function call with a Union type input into a case statement, where each branch of the case corresponds to + * one of the subtypes of the Union type. The function call is materialized in each of the branches, with the union input cast + * to the specific type corresponding to the branch of the case statement + * @param call + * @param functionLookupContext + * @return + */ private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookupContext functionLookupContext) { LogicalExpression[] args = new LogicalExpression[call.args.size()]; call.args.toArray(args); @@ -381,13 +387,18 @@ private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookup for (MinorType minorType : subTypes) { LogicalExpression ifCondition = getIsTypeExpressionForType(minorType, arg.accept(new CloneVisitor(), null)); - args[i] = getUnionCastExpressionForType(minorType, arg.accept(new CloneVisitor(), null)); + args[i] = getUnionAssertFunctionForType(minorType, arg.accept(new CloneVisitor(), null)); List newArgs = Lists.newArrayList(); for (LogicalExpression e : args) { newArgs.add(e.accept(new CloneVisitor(), null)); } + // When expanding the expression tree to handle the different subtypes, we will not throw an exception if one + // of the branches fails to find a function match, since it is possible that code path will never occur in execution + // So instead of failing to materialize, we generate code to throw the exception during execution if that code + // path is hit. + errorCollectors.push(errorCollector); errorCollector = new ErrorCollectorImpl(); @@ -415,6 +426,11 @@ private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookup throw new UnsupportedOperationException("Did not find any Union input types"); } + /** + * Returns the function call whose purpose is to throw an Exception if that code is hit during execution + * @param message the exception message + * @return + */ private LogicalExpression getExceptionFunction(String message) { QuotedString msg = new QuotedString(message, ExpressionPosition.UNKNOWN); List args = Lists.newArrayList(); @@ -423,7 +439,14 @@ private LogicalExpression getExceptionFunction(String message) { return call; } - private LogicalExpression getUnionCastExpressionForType(MinorType type, LogicalExpression arg) { + /** + * Returns the function which asserts that the current subtype of a union type is a specific type, and allows the materializer + * to bind to that specific type when doing function resolution + * @param type + * @param arg + * @return + */ + private LogicalExpression getUnionAssertFunctionForType(MinorType type, LogicalExpression arg) { if (type == MinorType.UNION) { return arg; } @@ -431,11 +454,16 @@ private LogicalExpression getUnionCastExpressionForType(MinorType type, LogicalE return getExceptionFunction("Unable to cast union to " + type); } String castFuncName = String.format("assert_%s", type.toString()); - List args = Lists.newArrayList(); - args.add(arg); - return new FunctionCall(castFuncName, args, ExpressionPosition.UNKNOWN); + Collections.singletonList(arg); + return new FunctionCall(castFuncName, Collections.singletonList(arg), ExpressionPosition.UNKNOWN); } + /** + * Get the function that tests whether a union type is a specific type + * @param type + * @param arg + * @return + */ private LogicalExpression getIsTypeExpressionForType(MinorType type, LogicalExpression arg) { String isFuncName = String.format("is_%s", type.toString()); List args = Lists.newArrayList(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java index 04b75baa3bb..9e5fced2145 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ExceptionFunction.java @@ -23,12 +23,13 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.Output; import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers; import org.apache.drill.exec.expr.holders.BigIntHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; public class ExceptionFunction { - public static final String EXCEPTION_FUNCTION_NAME = "throwException"; + public static final String EXCEPTION_FUNCTION_NAME = "__throwException"; @FunctionTemplate(name = EXCEPTION_FUNCTION_NAME, isRandom = true, scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) @@ -41,9 +42,8 @@ public void setup() { } public void eval() { - byte[] bytes = new byte[message.end - message.start]; - message.buffer.getBytes(message.start, bytes, 0, message.end - message.start); - org.apache.drill.exec.expr.fn.ExceptionFunction.throwException(new String(bytes)); + String msg = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(message.start, message.end, message.buffer); + org.apache.drill.exec.expr.fn.ExceptionFunction.throwException(msg); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java index 9e9a6e3020a..75b70090f24 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/UnionFunctions.java @@ -36,6 +36,9 @@ import javax.inject.Inject; +/** + * The class contains additional functions for union types in addition to those in GUnionFunctions + */ public class UnionFunctions { @FunctionTemplate(names = {"typeOf"}, @@ -85,7 +88,7 @@ public void eval() { } @SuppressWarnings("unused") - @FunctionTemplate(name = "asList", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "ASSERT_LIST", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class CastUnionList implements DrillSimpleFunc { @Param UnionHolder in; @@ -95,6 +98,9 @@ public void setup() {} public void eval() { if (in.isSet == 1) { + if (in.reader.getType().getMinorType() != org.apache.drill.common.types.TypeProtos.MinorType.LIST) { + throw new UnsupportedOperationException("The input is not a LIST type"); + } out.reader = in.reader; } else { out.isSet = 0; @@ -103,7 +109,7 @@ public void eval() { } @SuppressWarnings("unused") - @FunctionTemplate(name = "is_LIST", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "IS_LIST", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class UnionIsList implements DrillSimpleFunc { @Param UnionHolder in; @@ -121,7 +127,7 @@ public void eval() { } @SuppressWarnings("unused") - @FunctionTemplate(name = "asMap", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "ASSERT_MAP", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class CastUnionMap implements DrillSimpleFunc { @Param UnionHolder in; @@ -131,6 +137,9 @@ public void setup() {} public void eval() { if (in.isSet == 1) { + if (in.reader.getType().getMinorType() != org.apache.drill.common.types.TypeProtos.MinorType.MAP) { + throw new UnsupportedOperationException("The input is not a MAP type"); + } out.reader = in.reader; } else { out.isSet = 0; @@ -139,7 +148,7 @@ public void eval() { } @SuppressWarnings("unused") - @FunctionTemplate(name = "is_MAP", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) + @FunctionTemplate(name = "IS_MAP", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL) public static class UnionIsMap implements DrillSimpleFunc { @Param UnionHolder in; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java index 98b28763d29..e7ee84d68bf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java @@ -90,6 +90,7 @@ public static RootExec getExec(FragmentContext context, FragmentRoot root) throw return rootExec; } catch(Exception e) { + e.printStackTrace(); context.fail(e); for(final CloseableRecordBatch crb : creator.getOperators()) { AutoCloseables.close(crb, logger); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java index e2052464833..8a7d705c5fc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java @@ -213,7 +213,7 @@ private HashAggregator createAggregatorInternal() throws SchemaChangeException, ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry()); if (expr instanceof IfExpression) { - throw new SchemaChangeException("Union type not supported in aggregate functions"); + throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger); } if (collector.hasErrors()) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java index 9f92d22d056..89122c8aafa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java @@ -282,7 +282,7 @@ private StreamingAggregator createAggregatorInternal() throws SchemaChangeExcept final NamedExpression ne = popConfig.getExprs()[i]; final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry()); if (expr instanceof IfExpression) { - throw new SchemaChangeException("Union type not supported in aggregate functions"); + throw UserException.unsupportedError(new UnsupportedOperationException("Union type not supported in aggregate functions")).build(logger); } if (expr == null) { continue; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index 9f64626e30d..3a8b73580cb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -288,7 +288,7 @@ protected boolean setupNewSchema() throws SchemaChangeException { final IntOpenHashSet transferFieldIds = new IntOpenHashSet(); final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn())); - final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression)ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true, false); + final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression)ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true); final TransferPair tp = getFlattenFieldTransferPair(flattenExpr.getRef()); if (tp != null) { @@ -315,7 +315,7 @@ protected boolean setupNewSchema() throws SchemaChangeException { } } - final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true, false); + final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true); final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType()); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index 2e3a403451c..2bffded612d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -104,10 +104,6 @@ private void clear() { } } - public static void crap() { - System.out.print(""); - } - public ProjectRecordBatch(final Project pop, final RecordBatch incoming, final FragmentContext context) throws OutOfMemoryException { super(pop, context, incoming); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java index ff04e0d7d48..a6294d89d05 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java @@ -59,7 +59,6 @@ public final int projectRecords(int startIndex, final int recordCount, int first final int countN = recordCount; int i; for (i = startIndex; i < startIndex + countN; i++, firstOutputIndex++) { - ProjectRecordBatch.crap(); doEval(i, firstOutputIndex); } if (i < startIndex + recordCount || startIndex > 0) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java index 1687c3d1a95..ed7da9b3ee7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java @@ -134,7 +134,6 @@ public IterOutcome next() { throw new IllegalStateException (String.format("Incoming batch of %s has size %d, which is beyond the limit of %d", incoming.getClass().getName(), incoming.getRecordCount(), MAX_BATCH_SIZE)); } -// BatchPrinter.printBatch(incoming); if (VALIDATE_VECTORS) { VectorValidator.validate(incoming); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java index 2a8e91fe2ed..a349d76d178 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java @@ -31,7 +31,7 @@ import org.apache.drill.exec.vector.complex.AbstractMapVector; import org.apache.drill.exec.vector.complex.ListVector; import org.apache.drill.exec.vector.complex.MapVector; -import org.apache.drill.exec.vector.complex.impl.UnionVector; +import org.apache.drill.exec.vector.complex.UnionVector; import java.util.ArrayList; import java.util.List; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java index ecf2596edba..8091c5621d5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/UnionSqlAccessor.java @@ -20,7 +20,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; -import org.apache.drill.exec.vector.complex.impl.UnionVector; +import org.apache.drill.exec.vector.complex.UnionVector; import org.apache.drill.exec.vector.complex.impl.UnionWriter; import org.apache.drill.exec.vector.complex.reader.FieldReader; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java index 17b42a133b4..5f6d0d24d60 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java @@ -111,84 +111,10 @@ protected T typeify(ValueVector v, Class clazz) { } public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) { - if (seg == null) { - if (addToBreadCrumb) { - builder.intermediateType(this.getField().getType()); - } - return builder.finalType(this.getField().getType()).build(); - } - - if (seg.isArray()) { - if (seg.isLastPath()) { - builder // - .withIndex() // - .finalType(getLastPathType()); - - // remainder starts with the 1st array segment in SchemaPath. - // only set remainder when it's the only array segment. - if (addToBreadCrumb) { - addToBreadCrumb = false; - builder.remainder(seg); - } - return builder.build(); - } else { - if (addToBreadCrumb) { - addToBreadCrumb = false; - builder.remainder(seg); - } - } - } else { - // name segment. - } - - VectorWithOrdinal vord = getChildVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath()); - if (vord == null) { - return null; - } - - ValueVector v = vord.vector; - if (addToBreadCrumb) { - builder.intermediateType(v.getField().getType()); - builder.addId(vord.ordinal); - } - - if (v instanceof AbstractContainerVector) { - // we're looking for a multi path. - AbstractContainerVector c = (AbstractContainerVector) v; - return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); - } else if(v instanceof ListVector) { - ListVector list = (ListVector) v; - return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); - } else { - if (seg.isNamed()) { - if(addToBreadCrumb) { - builder.intermediateType(v.getField().getType()); - } - builder.finalType(v.getField().getType()); - } else { - builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); - } - - if (seg.isLastPath()) { - return builder.build(); - } else { - PathSegment child = seg.getChild(); - if (child.isLastPath() && child.isArray()) { - if (addToBreadCrumb) { - builder.remainder(child); - } - builder.withIndex(); - builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); - return builder.build(); - } else { - logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong."); - return null; - } - } - } + return FieldIdUtil.getFieldIdIfMatches(this, builder, addToBreadCrumb, seg); } - private MajorType getLastPathType() { + MajorType getLastPathType() { if((this.getField().getType().getMinorType() == MinorType.LIST && this.getField().getType().getMode() == DataMode.REPEATED)) { // Use Repeated scalar type instead of Required List. VectorWithOrdinal vord = getChildVectorWithOrdinal(null); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java index 1706f5cf036..2c93c319f85 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java @@ -173,7 +173,7 @@ public T getChild(String name, Class clazz) { * * Note that this method does not enforce any vector type check nor throws a schema change exception. */ - public void putChild(String name, ValueVector vector) { + protected void putChild(String name, ValueVector vector) { putVector(name, vector); field.addChild(vector.getField()); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java new file mode 100644 index 00000000000..f709fec36f0 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex; + +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.exec.record.TypedFieldId; +import org.apache.drill.exec.vector.ValueVector; + +public class FieldIdUtil { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FieldIdUtil.class); + public static TypedFieldId getFieldIdIfMatches(ValueVector vector, TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) { + if (seg == null) { + if (addToBreadCrumb) { + builder.intermediateType(vector.getField().getType()); + } + return builder.finalType(vector.getField().getType()).build(); + } + + if (seg.isArray()) { + if (seg.isLastPath()) { + MajorType type; + if (vector instanceof AbstractContainerVector) { + type = ((AbstractContainerVector) vector).getLastPathType(); + } else if (vector instanceof ListVector) { + type = ((ListVector) vector).getDataVector().getField().getType(); + builder.listVector(); + } else { + throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType()); + } + builder // + .withIndex() // + .finalType(type); + + // remainder starts with the 1st array segment in SchemaPath. + // only set remainder when it's the only array segment. + if (addToBreadCrumb) { + addToBreadCrumb = false; + builder.remainder(seg); + } + return builder.build(); + } else { + if (addToBreadCrumb) { + addToBreadCrumb = false; + builder.remainder(seg); + } + } + } else { + if (vector instanceof ListVector) { + return null; + } + } + + ValueVector v; + if (vector instanceof AbstractContainerVector) { + VectorWithOrdinal vord = ((AbstractContainerVector) vector).getChildVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath()); + if (vord == null) { + return null; + } + v = vord.vector; + if (addToBreadCrumb) { + builder.intermediateType(v.getField().getType()); + builder.addId(vord.ordinal); + } + } else if (vector instanceof ListVector) { + v = ((ListVector) vector).getDataVector(); + } else { + throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType()); + } + + if (v instanceof AbstractContainerVector) { + // we're looking for a multi path. + AbstractContainerVector c = (AbstractContainerVector) v; + return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); + } else if(v instanceof ListVector) { + ListVector list = (ListVector) v; + return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); + } else { + if (seg.isNamed()) { + if(addToBreadCrumb) { + builder.intermediateType(v.getField().getType()); + } + builder.finalType(v.getField().getType()); + } else { + builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); + } + + if (seg.isLastPath()) { + return builder.build(); + } else { + PathSegment child = seg.getChild(); + if (child.isLastPath() && child.isArray()) { + if (addToBreadCrumb) { + builder.remainder(child); + } + builder.withIndex(); + builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); + return builder.build(); + } else { + logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong."); + return null; + } + } + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java index 9a0f4b754cc..e5d94430ec0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java @@ -34,7 +34,6 @@ import org.apache.drill.exec.util.CallBack; import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.vector.AddOrGetResult; -import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.UInt1Vector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; @@ -43,7 +42,6 @@ import org.apache.drill.exec.vector.complex.impl.ComplexCopier; import org.apache.drill.exec.vector.complex.impl.UnionListReader; import org.apache.drill.exec.vector.complex.impl.UnionListWriter; -import org.apache.drill.exec.vector.complex.impl.UnionVector; import org.apache.drill.exec.vector.complex.reader.FieldReader; import org.apache.drill.exec.vector.complex.writer.FieldWriter; @@ -96,8 +94,7 @@ public void copyFrom(int inIndex, int outIndex, ListVector from) { in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier copier = new ComplexCopier(in, out); - copier.write(); + ComplexCopier.copy(in, out); } @Override @@ -136,7 +133,10 @@ public void transfer() { @Override public void splitAndTransfer(int startIndex, int length) { - + to.allocateNew(); + for (int i = 0; i < length; i++) { + copyValueSafe(startIndex + i, i); + } } @Override @@ -184,8 +184,10 @@ public boolean allocateNewSafe() { clear(); } } - offsets.zeroVector(); - bits.zeroVector(); + if (success) { + offsets.zeroVector(); + bits.zeroVector(); + } return success; } @@ -262,72 +264,7 @@ public UnionVector promoteToUnion() { } public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) { - if (seg == null) { - if (addToBreadCrumb) { - builder.intermediateType(this.getField().getType()); - } - return builder.finalType(this.getField().getType()).build(); - } - - if (seg.isArray()) { - if (seg.isLastPath()) { - builder // - .withIndex() // - .listVector() - .finalType(getDataVector().getField().getType()); - - // remainder starts with the 1st array segment in SchemaPath. - // only set remainder when it's the only array segment. - if (addToBreadCrumb) { - addToBreadCrumb = false; - builder.remainder(seg); - } - return builder.build(); - } else { - if (addToBreadCrumb) { - addToBreadCrumb = false; - builder.remainder(seg); - } - } - } else { - return null; - } - - ValueVector v = getDataVector(); - if (v instanceof AbstractContainerVector) { - // we're looking for a multi path. - AbstractContainerVector c = (AbstractContainerVector) v; - return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); - } else if (v instanceof ListVector) { - ListVector list = (ListVector) v; - return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); - } else { - if (seg.isNamed()) { - if(addToBreadCrumb) { - builder.intermediateType(v.getField().getType()); - } - builder.finalType(v.getField().getType()); - } else { - builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); - } - - if (seg.isLastPath()) { - return builder.build(); - } else { - PathSegment child = seg.getChild(); - if (child.isLastPath() && child.isArray()) { - if (addToBreadCrumb) { - builder.remainder(child); - } - builder.withIndex(); - builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); - return builder.build(); - } else { -// logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong."); - return null; - } - } - } + return FieldIdUtil.getFieldIdIfMatches(this, builder, addToBreadCrumb, seg); } private int lastSet; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java index 7624b1d642e..05fe63f4ac2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java @@ -93,7 +93,6 @@ public void copyAsValue(UnionWriter writer) { @Override public void copyAsValue(ListWriter writer) { - ComplexCopier copier = new ComplexCopier(this, (FieldWriter)writer); - copier.write(); + ComplexCopier.copy(this, (FieldWriter)writer); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java index 23bcfcb635d..5575b5e92e7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java @@ -126,7 +126,7 @@ public MapWriter directMap(){ case INIT: MapVector map = (MapVector) container; - mapRoot = new SingleMapWriter(map, this, unionEnabled, false); + mapRoot = new SingleMapWriter(map, this, unionEnabled); mapRoot.setPosition(idx()); mode = Mode.MAP; break; @@ -147,7 +147,7 @@ public MapWriter rootAsMap() { case INIT: MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class); - mapRoot = new SingleMapWriter(map, this, unionEnabled, false); + mapRoot = new SingleMapWriter(map, this, unionEnabled); mapRoot.setPosition(idx()); mode = Mode.MAP; break; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java index d3459c3aed0..93ce5262348 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.vector.ZeroVector; import org.apache.drill.exec.vector.complex.AbstractMapVector; import org.apache.drill.exec.vector.complex.ListVector; +import org.apache.drill.exec.vector.complex.UnionVector; import org.apache.drill.exec.vector.complex.writer.FieldWriter; import java.lang.reflect.Constructor; @@ -92,7 +93,7 @@ private void setWriter(ValueVector v) { try { Constructor constructor = null; for (Constructor c : writerClass.getConstructors()) { - if (c.getParameterTypes().length == 4) { + if (c.getParameterTypes().length == 3) { constructor = c; } } @@ -100,7 +101,7 @@ private void setWriter(ValueVector v) { constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class); writer = (FieldWriter) constructor.newInstance(vector, null); } else { - writer = (FieldWriter) constructor.newInstance(vector, null, true, false); + writer = (FieldWriter) constructor.newInstance(vector, null, true); } } catch (ReflectiveOperationException e) { throw new RuntimeException(e); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java index db2d8e48512..2d351f270b5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java @@ -95,7 +95,6 @@ public boolean next() { } public void copyAsValue(ListWriter writer) { - ComplexCopier copier = new ComplexCopier(this, (FieldWriter) writer); - copier.write(); + ComplexCopier.copy(this, (FieldWriter) writer); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java index 6bc2e05a571..b19b0293d92 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java @@ -37,7 +37,7 @@ public VectorContainerWriter(OutputMutator mutator, boolean unionEnabled) { super(null); this.mutator = mutator; mapVector = new SpecialMapVector(mutator.getCallBack()); - mapRoot = new SingleMapWriter(mapVector, this, unionEnabled, false); + mapRoot = new SingleMapWriter(mapVector, this, unionEnabled); } public VectorContainerWriter(OutputMutator mutator) { diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index 05bd1387a97..55440f7f4bf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -34,15 +34,6 @@ public class TestExampleQueries extends BaseTestQuery { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class); - @After - public void reset() throws Exception { - String[] options = new String[] { ExecConstants.SLICE_TARGET, PlannerSettings.HASHJOIN.getOptionName(), PlannerSettings.HASHAGG.getOptionName(), - PlannerSettings.STREAMAGG.getOptionName(), PlannerSettings.MERGEJOIN.getOptionName(), PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR.getOptionName() }; - for (String option : options) { - testNoResult(String.format("reset `%s`", option)); - } - } - @Test // see DRILL-2328 public void testConcatOnNull() throws Exception { try {