From f72826afec378a0704d19a438e8de0df8706f9bf Mon Sep 17 00:00:00 2001 From: Hanifi Gunes Date: Tue, 23 Jun 2015 12:05:45 -0700 Subject: [PATCH 1/2] DRILL-3313: Eliminate redundant #load methods and unit-test loading & exporting of vectors --- .../codegen/templates/FixedValueVectors.java | 147 +++++++------- .../templates/NullableValueVectors.java | 181 ++++++------------ .../templates/VariableLengthVectors.java | 117 +++++------ .../apache/drill/exec/vector/BitVector.java | 23 +-- .../drill/exec/vector/FixedWidthVector.java | 18 +- .../drill/exec/vector/NullableVector.java | 2 +- .../exec/vector/VariableWidthVector.java | 27 +-- .../complex/BaseRepeatedValueVector.java | 2 +- .../drill/exec/vector/complex/MapVector.java | 5 +- .../vector/complex/RepeatedMapVector.java | 2 +- .../exec/record/vector/TestValueVector.java | 119 ++++++++++++ 11 files changed, 329 insertions(+), 314 deletions(-) diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java index 7bb76fa041c..0189510648b 100644 --- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java @@ -40,7 +40,7 @@ * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector { +public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this); @@ -55,22 +55,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public FieldReader getReader() { + public FieldReader getReader(){ return reader; } @Override - public int getValueCapacity() { - return (int) (data.capacity() * 1.0 / ${type.width}); + public int getValueCapacity(){ + return (int) (data.capacity() *1.0 / ${type.width}); } @Override - public Accessor getAccessor() { + public Accessor getAccessor(){ return accessor; } @Override - public Mutator getMutator() { + public Mutator getMutator(){ return mutator; } @@ -85,7 +85,7 @@ public void setInitialCapacity(final int valueCount) { @Override public void allocateNew() { - if(!allocateNewSafe()) { + if(!allocateNewSafe()){ throw new OutOfMemoryRuntimeException("Failure while allocating buffer."); } } @@ -101,7 +101,7 @@ public boolean allocateNewSafe() { allocationMonitor = 0; } - try { + try{ allocateBytes(curAllocationSize); } catch (DrillRuntimeException ex) { return false; @@ -147,14 +147,14 @@ public void reAlloc() { } logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize); - final DrillBuf newBuf = allocator.buffer((int) newAllocationSize); + final DrillBuf newBuf = allocator.buffer((int)newAllocationSize); newBuf.setBytes(0, data, 0, data.capacity()); final int halfNewCapacity = newBuf.capacity() / 2; newBuf.setZero(halfNewCapacity, halfNewCapacity); newBuf.writerIndex(data.writerIndex()); data.release(1); data = newBuf; - allocationSizeInBytes = (int) newAllocationSize; + allocationSizeInBytes = (int)newAllocationSize; } /** @@ -166,27 +166,22 @@ public void zeroVector() { } @Override - public int load(int valueCount, DrillBuf buf) { + public void load(SerializedField metadata, DrillBuf buffer) { + final int actualLength = metadata.getBufferLength(); + final int valueCount = metadata.getValueCount(); + final int expectedLength = valueCount * ${type.width}; + assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength); + clear(); - final int len = valueCount * ${type.width}; if (data != null) { data.release(1); } - data = buf.slice(0, len); + data = buffer.slice(0, actualLength); data.retain(1); - data.writerIndex(len); - return len; - } - - @Override - public void load(SerializedField metadata, DrillBuf buffer) { - assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata); - final int loaded = load(metadata.getValueCount(), buffer); - assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded); - } + data.writerIndex(actualLength); + } - @Override - public TransferPair getTransferPair() { + public TransferPair getTransferPair(){ return new TransferImpl(getField()); } @@ -200,7 +195,7 @@ public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((${minor.class}Vector) to); } - public void transferTo(${minor.class}Vector target) { + public void transferTo(${minor.class}Vector target){ target.clear(); target.data = data; target.data.retain(1); @@ -217,10 +212,10 @@ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target.data.writerIndex(sliceLength); } - private class TransferImpl implements TransferPair { + private class TransferImpl implements TransferPair{ private ${minor.class}Vector to; - public TransferImpl(MaterializedField field) { + public TransferImpl(MaterializedField field){ to = new ${minor.class}Vector(field, allocator); } @@ -229,12 +224,12 @@ public TransferImpl(${minor.class}Vector to) { } @Override - public ${minor.class}Vector getTo() { + public ${minor.class}Vector getTo(){ return to; } @Override - public void transfer() { + public void transfer(){ transferTo(to); } @@ -249,7 +244,7 @@ public void copyValueSafe(int fromIndex, int toIndex) { } } - public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) { + public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ <#if (type.width > 8)> from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width}); <#else> <#-- type.width <= 8 --> @@ -259,7 +254,7 @@ public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) { <#-- type.width --> } - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ while(thisIndex >= getValueCapacity()) { reAlloc(); } @@ -284,7 +279,7 @@ public int getValueCount() { } @Override - public boolean isNull(int index) { + public boolean isNull(int index){ return false; } @@ -295,7 +290,7 @@ public boolean isNull(int index) { } <#if (minor.class == "Interval")> - public void get(int index, ${minor.class}Holder holder) { + public void get(int index, ${minor.class}Holder holder){ final int offsetIndex = index * ${type.width}; holder.months = data.getInt(offsetIndex); @@ -303,7 +298,7 @@ public void get(int index, ${minor.class}Holder holder) { holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } - public void get(int index, Nullable${minor.class}Holder holder) { + public void get(int index, Nullable${minor.class}Holder holder){ final int offsetIndex = index * ${type.width}; holder.isSet = 1; holder.months = data.getInt(offsetIndex); @@ -357,14 +352,14 @@ public StringBuilder getAsStringBuilder(int index) { } <#elseif (minor.class == "IntervalDay")> - public void get(int index, ${minor.class}Holder holder) { + public void get(int index, ${minor.class}Holder holder){ final int offsetIndex = index * ${type.width}; holder.days = data.getInt(offsetIndex); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } - public void get(int index, Nullable${minor.class}Holder holder) { + public void get(int index, Nullable${minor.class}Holder holder){ final int offsetIndex = index * ${type.width}; holder.isSet = 1; holder.days = data.getInt(offsetIndex); @@ -434,12 +429,12 @@ public void get(int index, Nullable${minor.class}Holder holder) { } <#else> - public void get(int index, ${minor.class}Holder holder) { + public void get(int index, ${minor.class}Holder holder){ holder.buffer = data; holder.start = index * ${type.width}; } - public void get(int index, Nullable${minor.class}Holder holder) { + public void get(int index, Nullable${minor.class}Holder holder){ holder.isSet = 1; holder.buffer = data; holder.start = index * ${type.width}; @@ -534,7 +529,7 @@ public DateTime getObject(int index) { } - public void get(int index, ${minor.class}Holder holder) { + public void get(int index, ${minor.class}Holder holder){ <#if minor.class.startsWith("Decimal")> holder.scale = getField().getScale(); holder.precision = getField().getPrecision(); @@ -543,7 +538,7 @@ public void get(int index, ${minor.class}Holder holder) { holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); } - public void get(int index, Nullable${minor.class}Holder holder) { + public void get(int index, Nullable${minor.class}Holder holder){ holder.isSet = 1; holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); } @@ -563,7 +558,7 @@ public void get(int index, Nullable${minor.class}Holder holder) { */ public final class Mutator extends BaseDataValueVector.BaseMutator { - private Mutator() {}; + private Mutator(){}; /** * Set the element at the given index to the given value. Note that widths smaller than * 32 bits are handled by the DrillBuf interface. @@ -584,122 +579,122 @@ public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaT } <#if (minor.class == "Interval")> - public void set(int index, int months, int days, int milliseconds) { + public void set(int index, int months, int days, int milliseconds){ final int offsetIndex = index * ${type.width}; data.setInt(offsetIndex, months); data.setInt((offsetIndex + ${minor.daysOffset}), days); data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); } - protected void set(int index, ${minor.class}Holder holder) { + protected void set(int index, ${minor.class}Holder holder){ set(index, holder.months, holder.days, holder.milliseconds); } - protected void set(int index, Nullable${minor.class}Holder holder) { + protected void set(int index, Nullable${minor.class}Holder holder){ set(index, holder.months, holder.days, holder.milliseconds); } - public void setSafe(int index, int months, int days, int milliseconds) { + public void setSafe(int index, int months, int days, int milliseconds){ while(index >= getValueCapacity()) { reAlloc(); } set(index, months, days, milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder) { + public void setSafe(int index, Nullable${minor.class}Holder holder){ setSafe(index, holder.months, holder.days, holder.milliseconds); } - public void setSafe(int index, ${minor.class}Holder holder) { + public void setSafe(int index, ${minor.class}Holder holder){ setSafe(index, holder.months, holder.days, holder.milliseconds); } <#elseif (minor.class == "IntervalDay")> - public void set(int index, int days, int milliseconds) { + public void set(int index, int days, int milliseconds){ final int offsetIndex = index * ${type.width}; data.setInt(offsetIndex, days); data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); } - protected void set(int index, ${minor.class}Holder holder) { + protected void set(int index, ${minor.class}Holder holder){ set(index, holder.days, holder.milliseconds); } - protected void set(int index, Nullable${minor.class}Holder holder) { + protected void set(int index, Nullable${minor.class}Holder holder){ set(index, holder.days, holder.milliseconds); } - public void setSafe(int index, int days, int milliseconds) { + public void setSafe(int index, int days, int milliseconds){ while(index >= getValueCapacity()) { reAlloc(); } set(index, days, milliseconds); } - public void setSafe(int index, ${minor.class}Holder holder) { + public void setSafe(int index, ${minor.class}Holder holder){ setSafe(index, holder.days, holder.milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder) { + public void setSafe(int index, Nullable${minor.class}Holder holder){ setSafe(index, holder.days, holder.milliseconds); } <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")> - public void set(int index, ${minor.class}Holder holder) { + public void set(int index, ${minor.class}Holder holder){ set(index, holder.start, holder.buffer); } - void set(int index, Nullable${minor.class}Holder holder) { + void set(int index, Nullable${minor.class}Holder holder){ set(index, holder.start, holder.buffer); } - public void setSafe(int index, Nullable${minor.class}Holder holder) { + public void setSafe(int index, Nullable${minor.class}Holder holder){ setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, ${minor.class}Holder holder) { + public void setSafe(int index, ${minor.class}Holder holder){ setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, int start, DrillBuf buffer) { + public void setSafe(int index, int start, DrillBuf buffer){ while(index >= getValueCapacity()) { reAlloc(); } set(index, start, buffer); } - public void set(int index, int start, DrillBuf buffer) { + public void set(int index, int start, DrillBuf buffer){ data.setBytes(index * ${type.width}, buffer, start, ${type.width}); } <#else> - protected void set(int index, ${minor.class}Holder holder) { + protected void set(int index, ${minor.class}Holder holder){ set(index, holder.start, holder.buffer); } - public void set(int index, Nullable${minor.class}Holder holder) { + public void set(int index, Nullable${minor.class}Holder holder){ set(index, holder.start, holder.buffer); } - public void set(int index, int start, DrillBuf buffer) { + public void set(int index, int start, DrillBuf buffer){ data.setBytes(index * ${type.width}, buffer, start, ${type.width}); } - public void setSafe(int index, ${minor.class}Holder holder) { + public void setSafe(int index, ${minor.class}Holder holder){ setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, Nullable${minor.class}Holder holder) { + public void setSafe(int index, Nullable${minor.class}Holder holder){ setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, int start, DrillBuf buffer) { + public void setSafe(int index, int start, DrillBuf buffer){ while(index >= getValueCapacity()) { reAlloc(); } set(index, holder); } - public void set(int index, Nullable${minor.class}Holder holder) { + public void set(int index, Nullable${minor.class}Holder holder){ data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width}); } @@ -711,7 +706,7 @@ public void generateTestData(int count) { final int valueCount = getAccessor().getValueCount(); for(int i = 0; i < valueCount; i++, even = !even) { final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE; - for(int w = 0; w < ${type.width}; w++) { + for(int w = 0; w < ${type.width}; w++){ data.setByte(i + w, b); } } @@ -729,22 +724,22 @@ public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.java set(index, value); } - protected void set(int index, ${minor.class}Holder holder) { + protected void set(int index, ${minor.class}Holder holder){ data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); } - public void setSafe(int index, ${minor.class}Holder holder) { + public void setSafe(int index, ${minor.class}Holder holder){ while(index >= getValueCapacity()) { reAlloc(); } set(index, holder); } - protected void set(int index, Nullable${minor.class}Holder holder) { + protected void set(int index, Nullable${minor.class}Holder holder){ data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); } - public void setSafe(int index, Nullable${minor.class}Holder holder) { + public void setSafe(int index, Nullable${minor.class}Holder holder){ while(index >= getValueCapacity()) { reAlloc(); } @@ -757,9 +752,9 @@ public void generateTestData(int size) { boolean even = true; final int valueCount = getAccessor().getValueCount(); for(int i = 0; i < valueCount; i++, even = !even) { - if (even) { + if(even){ set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE); - } else { + }else{ set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE); } } @@ -770,9 +765,9 @@ public void generateTestDataAlt(int size) { boolean even = true; final int valueCount = getAccessor().getValueCount(); for(int i = 0; i < valueCount; i++, even = !even) { - if (even) { + if(even){ set(i, (${(minor.javaType!type.javaType)}) 1); - } else { + }else{ set(i, (${(minor.javaType!type.javaType)}) 0); } } diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java index 4bdde163f3e..fe4f71fa0b5 100644 --- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java @@ -45,13 +45,15 @@ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @SuppressWarnings("unused") -public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, NullableVector { +public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, NullableVector{ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this); - private final UInt1Vector bits = new UInt1Vector(MaterializedField.create(field + "_bits", Types.required(MinorType.UINT1)), allocator); + private final MaterializedField bitsField = MaterializedField.create("$bits$", Types.required(MinorType.UINT1)); + private final UInt1Vector bits = new UInt1Vector(bitsField, allocator); private final ${valuesName} values = new ${minor.class}Vector(field, allocator); + private final Mutator mutator = new Mutator(); private final Accessor accessor = new Accessor(); @@ -60,12 +62,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type } @Override - public FieldReader getReader() { + public FieldReader getReader(){ return reader; } @Override - public int getValueCapacity() { + public int getValueCapacity(){ return Math.min(bits.getValueCapacity(), values.getValueCapacity()); } @@ -96,7 +98,7 @@ public void clear() { } @Override - public int getBufferSize() { + public int getBufferSize(){ return values.getBufferSize() + bits.getBufferSize(); } @@ -116,19 +118,16 @@ public void setInitialCapacity(int numRecords) { values.setInitialCapacity(numRecords); } - <#if type.major == "VarLen"> @Override - public SerializedField getMetadata() { - return getMetadataBuilder() - .setValueCount(getAccessor().getValueCount()) - .setVarByteLength(values.getVarByteLength()) - .setBufferLength(getBufferSize()) - .build(); + public SerializedField.Builder getMetadataBuilder() { + return super.getMetadataBuilder() + .addChild(bits.getMetadata()) + .addChild(values.getMetadata()); } @Override public void allocateNew() { - if(!allocateNewSafe()) { + if(!allocateNewSafe()){ throw new OutOfMemoryRuntimeException("Failure while allocating buffer."); } } @@ -154,6 +153,7 @@ public boolean allocateNewSafe() { return success; } + <#if type.major == "VarLen"> @Override public void allocateNew(int totalBytes, int valueCount) { try { @@ -169,78 +169,21 @@ public void allocateNew(int totalBytes, int valueCount) { } @Override - public int load(int dataBytes, int valueCount, DrillBuf buf) { - clear(); - int loaded = bits.load(valueCount, buf); - - // remove bits part of buffer. - buf = buf.slice(loaded, buf.capacity() - loaded); - dataBytes -= loaded; - loaded += values.load(dataBytes, valueCount, buf); - mutator.lastSet = valueCount; - return loaded; - } - - @Override - public void load(SerializedField metadata, DrillBuf buffer) { - assert field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", field, metadata); - final int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer); - assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded); - } - - @Override - public int getByteCapacity() { + public int getByteCapacity(){ return values.getByteCapacity(); } @Override - public int getCurrentSizeInBytes() { + public int getCurrentSizeInBytes(){ return values.getCurrentSizeInBytes(); } <#else> - - @Override - public void allocateNew() { - try { - values.allocateNew(); - bits.allocateNew(); - } catch(DrillRuntimeException e) { - clear(); - throw e; - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - } - - - @Override - public boolean allocateNewSafe() { - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - success = values.allocateNewSafe() && bits.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - return success; - } - @Override public void allocateNew(int valueCount) { try { values.allocateNew(valueCount); - bits.allocateNew(valueCount); + bits.allocateNew(valueCount+1); } catch(OutOfMemoryRuntimeException e) { clear(); throw e; @@ -255,32 +198,26 @@ public void allocateNew(int valueCount) { */ @Override public void zeroVector() { - values.zeroVector(); bits.zeroVector(); + values.zeroVector(); } + - @Override - public int load(int valueCount, DrillBuf buf) { - clear(); - int loaded = bits.load(valueCount, buf); - - // remove bits part of buffer. - buf = buf.slice(loaded, buf.capacity() - loaded); - loaded += values.load(valueCount, buf); - return loaded; - } @Override public void load(SerializedField metadata, DrillBuf buffer) { - assert field.matches(metadata); - final int loaded = load(metadata.getValueCount(), buffer); - assert metadata.getBufferLength() == loaded; - } + clear(); + final SerializedField bitsField = metadata.getChild(0); + bits.load(bitsField, buffer); - + final int capacity = buffer.capacity(); + final int bitsLength = bitsField.getBufferLength(); + final SerializedField valuesField = metadata.getChild(1); + values.load(valuesField, buffer.slice(bitsLength, capacity - bitsLength)); + } @Override - public TransferPair getTransferPair() { + public TransferPair getTransferPair(){ return new TransferImpl(getField()); } @@ -294,7 +231,7 @@ public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((Nullable${minor.class}Vector) to); } - public void transferTo(Nullable${minor.class}Vector target) { + public void transferTo(Nullable${minor.class}Vector target){ bits.transferTo(target.bits); values.transferTo(target.values); <#if type.major == "VarLen"> @@ -314,21 +251,21 @@ public void splitAndTransferTo(int startIndex, int length, Nullable${minor.class private class TransferImpl implements TransferPair { Nullable${minor.class}Vector to; - public TransferImpl(MaterializedField field) { + public TransferImpl(MaterializedField field){ to = new Nullable${minor.class}Vector(field, allocator); } - public TransferImpl(Nullable${minor.class}Vector to) { + public TransferImpl(Nullable${minor.class}Vector to){ this.to = to; } @Override - public Nullable${minor.class}Vector getTo() { + public Nullable${minor.class}Vector getTo(){ return to; } @Override - public void transfer() { + public void transfer(){ transferTo(to); } @@ -344,16 +281,16 @@ public void copyValueSafe(int fromIndex, int toIndex) { } @Override - public Accessor getAccessor() { + public Accessor getAccessor(){ return accessor; } @Override - public Mutator getMutator() { + public Mutator getMutator(){ return mutator; } - public ${minor.class}Vector convertToRequiredVector() { + public ${minor.class}Vector convertToRequiredVector(){ ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator); if (v.data != null) { v.data.release(1); @@ -364,14 +301,14 @@ public Mutator getMutator() { return v; } - public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from) { + public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){ final Accessor fromAccessor = from.getAccessor(); if (!fromAccessor.isNull(fromIndex)) { mutator.set(thisIndex, fromAccessor.get(fromIndex)); } } - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ <#if type.major == "VarLen"> mutator.fillEmpties(thisIndex); @@ -379,7 +316,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from bits.getMutator().setSafe(thisIndex, 1); } - public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){ <#if type.major == "VarLen"> mutator.fillEmpties(thisIndex); @@ -400,7 +337,7 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m */ public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { if (isNull(index)) { - throw new IllegalStateException("Can't get a null value"); + throw new IllegalStateException("Can't get a null value"); } return vAccessor.get(index); } @@ -410,12 +347,12 @@ public boolean isNull(int index) { return isSet(index) == 0; } - public int isSet(int index) { + public int isSet(int index){ return bAccessor.get(index); } <#if type.major == "VarLen"> - public long getStartEnd(int index) { + public long getStartEnd(int index){ return vAccessor.getStartEnd(index); } @@ -425,7 +362,7 @@ public int getValueLength(int index) { } - public void get(int index, Nullable${minor.class}Holder holder) { + public void get(int index, Nullable${minor.class}Holder holder){ vAccessor.get(index, holder); holder.isSet = bAccessor.get(index); @@ -439,7 +376,7 @@ public void get(int index, Nullable${minor.class}Holder holder) { public ${friendlyType} getObject(int index) { if (isNull(index)) { return null; - } else { + }else{ return vAccessor.getObject(index); } } @@ -448,33 +385,33 @@ public void get(int index, Nullable${minor.class}Holder holder) { public StringBuilder getAsStringBuilder(int index) { if (isNull(index)) { return null; - } else { + }else{ return vAccessor.getAsStringBuilder(index); } } @Override - public int getValueCount() { + public int getValueCount(){ return bits.getAccessor().getValueCount(); } - public void reset() {} + public void reset(){} } public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator { private int setCount; <#if type.major = "VarLen"> private int lastSet = -1; - private Mutator() { + private Mutator(){ } - public ${valuesName} getVectorWithValues() { + public ${valuesName} getVectorWithValues(){ return values; } @Override - public void setIndexDefined(int index) { + public void setIndexDefined(int index){ bits.getMutator().set(index, 1); } @@ -500,9 +437,9 @@ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.widt <#if type.major == "VarLen"> - private void fillEmpties(int index) { + private void fillEmpties(int index){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); - for(int i = lastSet; i < index; i++) { + for (int i = lastSet; i < index; i++) { valuesMutator.setSafe(i + 1, emptyByteArray); } while(index > bits.getValueCapacity()) { @@ -543,20 +480,20 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { } - public void setNull(int index) { + public void setNull(int index){ bits.getMutator().setSafe(index, 0); } - public void setSkipNull(int index, ${minor.class}Holder holder) { + public void setSkipNull(int index, ${minor.class}Holder holder){ values.getMutator().set(index, holder); } - public void setSkipNull(int index, Nullable${minor.class}Holder holder) { + public void setSkipNull(int index, Nullable${minor.class}Holder holder){ values.getMutator().set(index, holder); } - public void set(int index, Nullable${minor.class}Holder holder) { + public void set(int index, Nullable${minor.class}Holder holder){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -568,7 +505,7 @@ public void set(int index, Nullable${minor.class}Holder holder) { <#if type.major == "VarLen">lastSet = index; } - public void set(int index, ${minor.class}Holder holder) { + public void set(int index, ${minor.class}Holder holder){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -585,7 +522,7 @@ public boolean isSafe(int outIndex) { } <#assign fields = minor.fields!type.fields /> - public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field ) { + public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field ){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -654,7 +591,7 @@ public void setValueCount(int valueCount) { } @Override - public void generateTestData(int valueCount) { + public void generateTestData(int valueCount){ bits.getMutator().generateTestDataAlt(valueCount); values.getMutator().generateTestData(valueCount); <#if type.major = "VarLen">lastSet = valueCount; @@ -662,7 +599,7 @@ public void generateTestData(int valueCount) { } @Override - public void reset() { + public void reset(){ setCount = 0; <#if type.major = "VarLen">lastSet = -1; } diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java index ac4d5390a0d..76fbed3a4fb 100644 --- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java @@ -48,17 +48,16 @@ * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector { +public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); private static final int DEFAULT_RECORD_BYTE_COUNT = 8; private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT; private static final int MIN_BYTE_COUNT = 4096; - public final static String OFFSETS_VECTOR_NAME = "offsets"; - private final static MaterializedField offsetsField = - MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4)); - private final UInt${type.width}Vector offsetVector; + public final static String OFFSETS_VECTOR_NAME = "$offsets$"; + private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4)); + private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator); private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this); private final Accessor accessor; @@ -71,19 +70,18 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) { super(field, allocator); - this.offsetVector = new UInt${type.width}Vector(offsetsField, allocator); this.oAccessor = offsetVector.getAccessor(); this.accessor = new Accessor(); this.mutator = new Mutator(); } @Override - public FieldReader getReader() { + public FieldReader getReader(){ return reader; } @Override - public int getBufferSize() { + public int getBufferSize(){ if (getAccessor().getValueCount() == 0) { return 0; } @@ -91,12 +89,12 @@ public int getBufferSize() { } @Override - public int getValueCapacity() { - return offsetVector.getValueCapacity() - 1; + public int getValueCapacity(){ + return Math.max(offsetVector.getValueCapacity() - 1, 0); } @Override - public int getByteCapacity() { + public int getByteCapacity(){ return data.capacity(); } @@ -109,7 +107,7 @@ public int getCurrentSizeInBytes() { * Return the number of bytes contained in the current var len byte vector. * @return */ - public int getVarByteLength() { + public int getVarByteLength(){ final int valueCount = getAccessor().getValueCount(); if(valueCount == 0) { return 0; @@ -119,31 +117,22 @@ public int getVarByteLength() { @Override public SerializedField getMetadata() { - return getMetadataBuilder() - .setValueCount(getAccessor().getValueCount()) - .setVarByteLength(getVarByteLength()) - .setBufferLength(getBufferSize()) + return getMetadataBuilder() // + .addChild(offsetVector.getMetadata()) + .setValueCount(getAccessor().getValueCount()) // + .setBufferLength(getBufferSize()) // .build(); } - @Override - public int load(int dataBytes, int valueCount, DrillBuf buf) { - if(valueCount == 0) { - allocateNew(0, 0); - return 0; - } - clear(); - final int loaded = offsetVector.load(valueCount + 1, buf); - data = buf.slice(loaded, dataBytes - loaded); - data.retain(1); - return dataBytes; - } - @Override public void load(SerializedField metadata, DrillBuf buffer) { - assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata); - int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer); - assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded); + final SerializedField offsetField = metadata.getChild(0); + offsetVector.load(offsetField, buffer); + + final int capacity = buffer.capacity(); + final int offsetsLength = offsetField.getBufferLength(); + data = buffer.slice(offsetsLength, capacity - offsetsLength); + data.retain(); } @Override @@ -165,16 +154,16 @@ public DrillBuf[] getBuffers(boolean clear) { return buffers; } - public long getOffsetAddr() { + public long getOffsetAddr(){ return offsetVector.getBuffer().memoryAddress(); } - public UInt${type.width}Vector getOffsetVector() { + public UInt${type.width}Vector getOffsetVector(){ return offsetVector; } @Override - public TransferPair getTransferPair() { + public TransferPair getTransferPair(){ return new TransferImpl(getField()); } @@ -188,7 +177,7 @@ public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((${minor.class}Vector) to); } - public void transferTo(${minor.class}Vector target) { + public void transferTo(${minor.class}Vector target){ target.clear(); this.offsetVector.transferTo(target.offsetVector); target.data = data; @@ -203,16 +192,16 @@ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target.clear(); target.offsetVector.allocateNew(length + 1); offsetVectorAccessor = this.offsetVector.getAccessor(); - final UInt4Vector.Mutator offsetVectorMutator = target.offsetVector.getMutator(); + final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator(); for (int i = 0; i < length + 1; i++) { - offsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint); + targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint); } target.data = data.slice(startPoint, sliceLength); target.data.retain(1); target.getMutator().setValueCount(length); } - protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) { + protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); final int start = fromOffsetVectorAccessor.get(fromIndex); final int end = fromOffsetVectorAccessor.get(fromIndex + 1); @@ -223,7 +212,7 @@ protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len); } - public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { + public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); final int start = fromOffsetVectorAccessor.get(fromIndex); final int end = fromOffsetVectorAccessor.get(fromIndex + 1); @@ -231,7 +220,7 @@ public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector f final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); while(data.capacity() < outputStart + len) { - reAlloc(); + reAlloc(); } offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len); @@ -239,24 +228,24 @@ public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector f return true; } - private class TransferImpl implements TransferPair { + private class TransferImpl implements TransferPair{ ${minor.class}Vector to; - public TransferImpl(MaterializedField field) { + public TransferImpl(MaterializedField field){ to = new ${minor.class}Vector(field, allocator); } - public TransferImpl(${minor.class}Vector to) { + public TransferImpl(${minor.class}Vector to){ this.to = to; } @Override - public ${minor.class}Vector getTo() { + public ${minor.class}Vector getTo(){ return to; } @Override - public void transfer() { + public void transfer(){ transferTo(to); } @@ -283,7 +272,7 @@ public void setInitialCapacity(final int valueCount) { @Override public void allocateNew() { - if(!allocateNewSafe()) { + if(!allocateNewSafe()){ throw new OutOfMemoryRuntimeException("Failure while allocating buffer."); } } @@ -310,7 +299,7 @@ public boolean allocateNewSafe() { * clear all the memory that we allocated */ try { - final int requestedSize = (int) curAllocationSize; + final int requestedSize = (int)curAllocationSize; data = allocator.buffer(requestedSize); offsetVector.allocateNew(); } catch (OutOfMemoryRuntimeException e) { @@ -363,7 +352,7 @@ private void incrementAllocationMonitor() { } @Override - public Accessor getAccessor() { + public Accessor getAccessor(){ return accessor; } @@ -374,7 +363,7 @@ public Mutator getMutator() { public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor { final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor(); - public long getStartEnd(int index) { + public long getStartEnd(int index){ return oAccessor.getTwoAsLong(index); } @@ -394,13 +383,13 @@ public int getValueLength(int index) { return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index); } - public void get(int index, ${minor.class}Holder holder) { + public void get(int index, ${minor.class}Holder holder){ holder.start = oAccessor.get(index); holder.end = oAccessor.get(index + 1); holder.buffer = data; } - public void get(int index, Nullable${minor.class}Holder holder) { + public void get(int index, Nullable${minor.class}Holder holder){ holder.isSet = 1; holder.start = oAccessor.get(index); holder.end = oAccessor.get(index + 1); @@ -436,11 +425,11 @@ public int getValueCount() { } @Override - public boolean isNull(int index) { + public boolean isNull(int index){ return false; } - public UInt${type.width}Vector getOffsetVector() { + public UInt${type.width}Vector getOffsetVector(){ return offsetVector; } } @@ -474,7 +463,7 @@ public void setSafe(int index, byte[] bytes) { assert index >= 0; final int currentOffset = offsetVector.getAccessor().get(index); - while(data.capacity() < currentOffset + bytes.length) { + while (data.capacity() < currentOffset + bytes.length) { reAlloc(); } offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length); @@ -530,7 +519,7 @@ public void setValueLengthSafe(int index, int length) { } - public void setSafe(int index, int start, int end, DrillBuf buffer) { + public void setSafe(int index, int start, int end, DrillBuf buffer){ final int len = end - start; final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); @@ -542,7 +531,7 @@ public void setSafe(int index, int start, int end, DrillBuf buffer) { buffer.getBytes(start, data, outputStart, len); } - public void setSafe(int index, Nullable${minor.class}Holder holder) { + public void setSafe(int index, Nullable${minor.class}Holder holder){ assert holder.isSet == 1; final int start = holder.start; @@ -559,7 +548,7 @@ public void setSafe(int index, Nullable${minor.class}Holder holder) { offsetVector.getMutator().setSafe( index+1, outputStart + len); } - public void setSafe(int index, ${minor.class}Holder holder) { + public void setSafe(int index, ${minor.class}Holder holder){ final int start = holder.start; final int end = holder.end; final int len = end - start; @@ -573,7 +562,7 @@ public void setSafe(int index, ${minor.class}Holder holder) { offsetVector.getMutator().setSafe( index+1, outputStart + len); } - protected void set(int index, int start, int length, DrillBuf buffer) { + protected void set(int index, int start, int length, DrillBuf buffer){ assert index >= 0; final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + length); @@ -581,14 +570,14 @@ protected void set(int index, int start, int length, DrillBuf buffer) { data.setBytes(currentOffset, bb); } - protected void set(int index, Nullable${minor.class}Holder holder) { + protected void set(int index, Nullable${minor.class}Holder holder){ final int length = holder.end - holder.start; final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + length); data.setBytes(currentOffset, holder.buffer, holder.start, length); } - protected void set(int index, ${minor.class}Holder holder) { + protected void set(int index, ${minor.class}Holder holder){ final int length = holder.end - holder.start; final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + length); @@ -610,7 +599,7 @@ public void setValueCount(int valueCount) { } @Override - public void generateTestData(int size) { + public void generateTestData(int size){ boolean even = true; <#switch minor.class> <#case "Var16Char"> @@ -622,9 +611,9 @@ public void generateTestData(int size) { final byte[] evenValue = new String("aaaaa").getBytes(charset); final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset); - for(int i =0; i < size; i++, even = !even) { + for(int i =0; i < size; i++, even = !even){ set(i, even ? evenValue : oddValue); - } + } setValueCount(size); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java index 054ef826123..65e88e601a1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java @@ -152,16 +152,6 @@ public void zeroVector() { data.setZero(0, data.capacity()); } - @Override - public int load(int valueCount, DrillBuf buf) { - clear(); - this.valueCount = valueCount; - int len = getSizeFromCount(valueCount); - data = buf.slice(0, len); - data.retain(); - return len; - } - public void copyFrom(int inIndex, int outIndex, BitVector from) { this.mutator.set(outIndex, from.accessor.get(inIndex)); } @@ -177,9 +167,16 @@ public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) { @Override public void load(SerializedField metadata, DrillBuf buffer) { - assert this.field.matches(metadata); - int loaded = load(metadata.getValueCount(), buffer); - assert metadata.getBufferLength() == loaded; + assert field.matches(metadata); + final int valueCount = metadata.getValueCount(); + final int expectedLength = getSizeFromCount(valueCount); + final int actualLength = metadata.getBufferLength(); + assert expectedLength == actualLength: "expected and actual buffer sizes do not match"; + + clear(); + data = buffer.slice(0, actualLength); + data.retain(); + this.valueCount = valueCount; } public Mutator getMutator() { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java index b546b9efa32..529c05483b2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.DrillBuf; public interface FixedWidthVector extends ValueVector{ @@ -26,21 +25,8 @@ public interface FixedWidthVector extends ValueVector{ * * @param valueCount Number of values in the vector. */ - public void allocateNew(int valueCount); - - /** - * Load the records in the provided buffer based on the given number of values. - * @param valueCount Number of values the buffer contains. - * @param buf Incoming buffer. - * @return The number of bytes of the buffer that were consumed. - */ - public int load(int valueCount, DrillBuf buf); + void allocateNew(int valueCount); + void zeroVector(); - public abstract Mutator getMutator(); - - /** - * Zero out the underlying buffer backing this vector. - */ - public void zeroVector(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java index 92f60d66eef..8091c4c56ac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java @@ -19,5 +19,5 @@ public interface NullableVector extends ValueVector{ - public ValueVector getValuesVector(); + ValueVector getValuesVector(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java index 10ddcc9ceec..ee9c0396e58 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java @@ -27,34 +27,25 @@ public interface VariableWidthVector extends ValueVector{ * @param totalBytes Desired size of the underlying data buffer. * @param valueCount Number of values in the vector. */ - public void allocateNew(int totalBytes, int valueCount); + void allocateNew(int totalBytes, int valueCount); /** * Provide the maximum amount of variable width bytes that can be stored int his vector. * @return */ - public int getByteCapacity(); + int getByteCapacity(); - /** - * Load the records in the provided buffer based on the given number of values. - * @param dataBytes The number of bytes associated with the data array. - * @param valueCount Number of values the buffer contains. - * @param buf Incoming buffer. - * @return The number of bytes of the buffer that were consumed. - */ - public int load(int dataBytes, int valueCount, DrillBuf buf); - - public abstract VariableWidthMutator getMutator(); + VariableWidthMutator getMutator(); - public abstract VariableWidthAccessor getAccessor(); + VariableWidthAccessor getAccessor(); - public interface VariableWidthAccessor extends Accessor { - public int getValueLength(int index); + interface VariableWidthAccessor extends Accessor { + int getValueLength(int index); } - public int getCurrentSizeInBytes(); + int getCurrentSizeInBytes(); - public interface VariableWidthMutator extends Mutator { - public void setValueLengthSafe(int index, int length); + interface VariableWidthMutator extends Mutator { + void setValueLengthSafe(int index, int length); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java index 9bf6d8572d3..fcec2ab06be 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java @@ -100,7 +100,7 @@ public void setInitialCapacity(int numRecords) { @Override public int getValueCapacity() { - final int offsetValueCapacity = offsets.getValueCapacity() - 1; + final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); if (vector == DEFAULT_DATA_VECTOR) { return offsetValueCapacity; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index 1e30ea2117f..a06ce660513 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -33,6 +33,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.expr.holders.ComplexHolder; import org.apache.drill.exec.memory.BufferAllocator; @@ -52,7 +53,7 @@ public class MapVector extends AbstractMapVector { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); - public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REQUIRED).build(); + public final static MajorType TYPE = Types.required(MinorType.MAP); private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this); private final Accessor accessor = new Accessor(); @@ -224,7 +225,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public int getValueCapacity() { if (size() == 0) { - return Integer.MAX_VALUE; + return 0; } final Ordering natural = new Ordering() { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index 644e5db67e5..40d0be4c964 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -415,7 +415,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) { @Override public int getValueCapacity() { - return offsets.getValueCapacity()-1; + return Math.max(offsets.getValueCapacity() - 1, 0); } @Override diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java index 4eeb8f282e0..339dbe8b0db 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java @@ -23,8 +23,13 @@ import java.nio.charset.Charset; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.netty.buffer.DrillBuf; import org.apache.drill.common.AutoCloseables; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.exception.OversizedAllocationException; import org.apache.drill.exec.expr.TypeHelper; @@ -36,9 +41,11 @@ import org.apache.drill.exec.expr.holders.NullableVarCharHolder; import org.apache.drill.exec.expr.holders.RepeatedFloat4Holder; import org.apache.drill.exec.expr.holders.RepeatedVarBinaryHolder; +import org.apache.drill.exec.expr.holders.UInt1Holder; import org.apache.drill.exec.expr.holders.UInt4Holder; import org.apache.drill.exec.expr.holders.VarCharHolder; import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.BitVector; @@ -480,4 +487,116 @@ public void testVVInitialCapacity() throws Exception { AutoCloseables.close(valueVectors); } } + + protected interface VectorVerifier { + void verify(ValueVector vector) throws Exception; + } + + protected static class ChildVerifier implements VectorVerifier { + public final TypeProtos.MajorType[] types; + + public ChildVerifier(TypeProtos.MajorType... childTypes) { + this.types = Preconditions.checkNotNull(childTypes); + } + + @Override + public void verify(ValueVector vector) throws Exception { + final String hint = String.format("%s failed the test case", vector.getClass().getSimpleName()); + + final UserBitShared.SerializedField metadata = vector.getMetadata(); + final int actual = metadata.getChildCount(); + assertEquals(hint, types.length, actual); + + for (int i = 0; i < types.length; i++) { + final UserBitShared.SerializedField child = metadata.getChild(i); + + assertEquals(hint, types[i], child.getMajorType()); + } + } + } + + /** + * Convenience method that allows running tests on various {@link ValueVector vector} instances. + * + * @param test test function to execute + */ + private void testVectors(VectorVerifier test) throws Exception { + final MaterializedField[] fields = { + MaterializedField.create(EMPTY_SCHEMA_PATH, UInt1Holder.TYPE), + MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE), + MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE), + MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE), + MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE), + MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE), + MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE) + }; + + final ValueVector[] vectors = { + new UInt4Vector(fields[0], allocator), + new BitVector(fields[1], allocator), + new VarCharVector(fields[2], allocator), + new NullableVarCharVector(fields[3], allocator), + new RepeatedListVector(fields[4], allocator, null), + new MapVector(fields[5], allocator, null), + new RepeatedMapVector(fields[6], allocator, null) + }; + + try { + for (final ValueVector vector : vectors) { + test.verify(vector); + } + } finally { + AutoCloseables.close(vectors); + } + } + + @Test + public void testVectorMetadataIsAccurate() throws Exception { + final VectorVerifier noChild = new ChildVerifier(); + final VectorVerifier offsetChild = new ChildVerifier(UInt4Holder.TYPE); + + final ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put(UInt4Vector.class, noChild); + builder.put(BitVector.class, noChild); + builder.put(VarCharVector.class, offsetChild); + builder.put(NullableVarCharVector.class, new ChildVerifier(UInt1Holder.TYPE, Types.optional(TypeProtos.MinorType.VARCHAR))); + builder.put(RepeatedListVector.class, new ChildVerifier(UInt4Holder.TYPE, Types.LATE_BIND_TYPE)); + builder.put(MapVector.class, noChild); + builder.put(RepeatedMapVector.class, offsetChild); + final ImmutableMap children = builder.build(); + + testVectors(new VectorVerifier() { + + @Override + public void verify(ValueVector vector) throws Exception { + + final Class klazz = vector.getClass(); + final VectorVerifier verifier = children.get(klazz); + verifier.verify(vector); + } + }); + } + + @Test + public void testVectorCanLoadEmptyBuffer() throws Exception { + final DrillBuf empty = allocator.getEmpty(); + + testVectors(new VectorVerifier() { + + @Override + public void verify(ValueVector vector) { + final String hint = String.format("%s failed the test case", vector.getClass().getSimpleName()); + final UserBitShared.SerializedField metadata = vector.getMetadata(); + assertEquals(hint, 0, metadata.getBufferLength()); + assertEquals(hint, 0, metadata.getValueCount()); + + vector.load(metadata, empty); + + assertEquals(hint, 0, vector.getValueCapacity()); + assertEquals(hint, 0, vector.getAccessor().getValueCount()); + + vector.clear(); + } + }); + } } From 73d6b499f6bbe3f5a0503afc22c02500d848b7a0 Mon Sep 17 00:00:00 2001 From: Parth Chandra Date: Tue, 25 Aug 2015 11:39:21 -0700 Subject: [PATCH 2/2] DRILL-3313: Address review comments --- exec/java-exec/src/main/codegen/includes/vv_imports.ftl | 1 + .../src/main/codegen/templates/FixedValueVectors.java | 1 + .../src/main/codegen/templates/NullableValueVectors.java | 1 + .../src/main/codegen/templates/VariableLengthVectors.java | 1 + .../src/main/java/org/apache/drill/exec/vector/BitVector.java | 3 ++- .../java/org/apache/drill/exec/vector/FixedWidthVector.java | 3 +++ .../org/apache/drill/exec/record/vector/TestValueVector.java | 2 +- 7 files changed, 10 insertions(+), 2 deletions(-) diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl index 733e0a5973c..2fedfd018a6 100644 --- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl +++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl @@ -17,6 +17,7 @@ import com.google.common.collect.ObjectArrays; import com.google.common.base.Charsets; import com.google.common.collect.ObjectArrays; +import com.google.common.base.Preconditions; import io.netty.buffer.*; import org.apache.commons.lang3.ArrayUtils; diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java index 0189510648b..953198711c2 100644 --- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java @@ -167,6 +167,7 @@ public void zeroVector() { @Override public void load(SerializedField metadata, DrillBuf buffer) { + Preconditions.checkArgument(this.field.matches(metadata), "The field %s doesn't match the provided metadata %s.", this.field, metadata); final int actualLength = metadata.getBufferLength(); final int valueCount = metadata.getValueCount(); final int expectedLength = valueCount * ${type.width}; diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java index fe4f71fa0b5..259005f7c3c 100644 --- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java @@ -207,6 +207,7 @@ public void zeroVector() { @Override public void load(SerializedField metadata, DrillBuf buffer) { clear(); + // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant) final SerializedField bitsField = metadata.getChild(0); bits.load(bitsField, buffer); diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java index 76fbed3a4fb..c9dae773027 100644 --- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java @@ -126,6 +126,7 @@ public SerializedField getMetadata() { @Override public void load(SerializedField metadata, DrillBuf buffer) { + // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant) final SerializedField offsetField = metadata.getChild(0); offsetVector.load(offsetField, buffer); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java index 65e88e601a1..9d31ea84f03 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.vector; +import com.google.common.base.Preconditions; import io.netty.buffer.DrillBuf; import org.apache.drill.common.expression.FieldReference; @@ -167,7 +168,7 @@ public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) { @Override public void load(SerializedField metadata, DrillBuf buffer) { - assert field.matches(metadata); + Preconditions.checkArgument(this.field.matches(metadata), "The field %s doesn't match the provided metadata %s.", this.field, metadata); final int valueCount = metadata.getValueCount(); final int expectedLength = getSizeFromCount(valueCount); final int actualLength = metadata.getBufferLength(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java index 529c05483b2..c2781eb338b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java @@ -27,6 +27,9 @@ public interface FixedWidthVector extends ValueVector{ */ void allocateNew(int valueCount); +/** + * Zero out the underlying buffer backing this vector. + */ void zeroVector(); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java index 339dbe8b0db..4e299dc8eaf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java @@ -522,7 +522,7 @@ public void verify(ValueVector vector) throws Exception { */ private void testVectors(VectorVerifier test) throws Exception { final MaterializedField[] fields = { - MaterializedField.create(EMPTY_SCHEMA_PATH, UInt1Holder.TYPE), + MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE),