DRILL-2997: Remove references to groupCount from SerializedField

exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java

@@ -228,23 +228,7 @@ public void allocateNew(int totalBytes, int valueCount, int innerValueCount) {
     offsets.zeroVector();
     mutator.reset();
   }
-
-  @Override
-  public int load(int dataBytes, int valueCount, int innerValueCount, DrillBuf buf){
-    clear();
-    int loaded = 0;
-    loaded += offsets.load(valueCount+1, buf.slice(loaded, buf.capacity() - loaded));
-    loaded += values.load(dataBytes + 4*(innerValueCount + 1), innerValueCount, buf.slice(loaded, buf.capacity() - loaded));
-    return loaded;
-  }
-
-  @Override
-  public void load(SerializedField metadata, DrillBuf buffer) {
-    assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata);
-    int loaded = load(metadata.getVarByteLength(), metadata.getGroupCount(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
-  }
-
+
   public int getByteCapacity(){
     return values.getByteCapacity();
   }
@@ -255,7 +239,7 @@ public void allocateNew(int valueCount, int innerValueCount) {
     clear();
     /* boolean to keep track if all the memory allocation were successful
      * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
+     * buffers for multiple vectors. If one of the allocations failed we need to//
      * clear all the memory that we allocated
      */
     boolean success = false;
@@ -269,21 +253,6 @@ public void allocateNew(int valueCount, int innerValueCount) {
     offsets.zeroVector();
     mutator.reset();
   }
-
-  public int load(int valueCount, int innerValueCount, DrillBuf buf){
-    clear();
-    int loaded = 0;
-    loaded += offsets.load(valueCount+1, buf.slice(loaded, buf.capacity() - loaded));
-    loaded += values.load(innerValueCount, buf.slice(loaded, buf.capacity() - loaded));
-    return loaded;
-  }
-
-  @Override
-  public void load(SerializedField metadata, DrillBuf buffer) {
-    assert this.field.matches(metadata);
-    int loaded = load(metadata.getGroupCount(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
   </#if>
 
 

exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java

@@ -57,6 +57,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
   private static final int MIN_BYTE_COUNT = 4096;
 
+  public final static String OFFSETS_VECTOR_NAME = "offsets";
+  private final static MaterializedField offsetsField =
+    MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4));
   private final UInt${type.width}Vector offsetVector;
   private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
 
@@ -71,7 +74,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
 
   public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
     super(field, allocator);
-    this.offsetVector = new UInt${type.width}Vector(null, allocator);
+    this.offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
     this.oAccessor = offsetVector.getAccessor();
     this.accessor = new Accessor();
     this.mutator = new Mutator();

exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java

@@ -93,7 +93,7 @@ public boolean load(RecordBatchDef def, DrillBuf buf) throws SchemaChangeExcepti
           vector = TypeHelper.getNewVector(fieldDef, allocator);
         }
 
-        if (field.getValueCount() == 0 && (!field.hasGroupCount() || field.getGroupCount() == 0)) {
+        if (field.getValueCount() == 0) {
           AllocationHelper.allocate(vector, 0, 0, 0);
         } else {
           vector.load(field, buf.slice(bufOffset, field.getBufferLength()));

exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java

@@ -35,7 +35,7 @@ public abstract class BaseValueVector implements ValueVector {
   public static final int INITIAL_VALUE_ALLOCATION = 4096;
 
   protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
-    this.field = field;
+    this.field = Preconditions.checkNotNull(field, "field cannot be null");
     this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
   }
 

exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java

@@ -44,9 +44,11 @@ protected AbstractMapVector(MaterializedField field, BufferAllocator allocator,
     super(field, allocator, callBack);
     // create the hierarchy of the child vectors based on the materialized field
     for (MaterializedField child : field.getChildren()) {
-      String fieldName = child.getLastName();
-      ValueVector  v = TypeHelper.getNewVector(child, allocator, callBack);
-      putVector(fieldName, v);
+      if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
+        String fieldName = child.getLastName();
+        ValueVector v = TypeHelper.getNewVector(child, allocator, callBack);
+        putVector(fieldName, v);
+      }
     }
   }
 

exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java

@@ -40,10 +40,10 @@
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
   public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
-  public final static String OFFSETS_VECTOR_NAME = "offsets";
-  public final static String DATA_VECTOR_NAME = "data";
+  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+  public final static String DATA_VECTOR_NAME = "$data$";
 
-  private final static MaterializedField offsetsField =
+  public final static MaterializedField OFFSETS_FIELD =
       MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4));
 
   protected final UInt4Vector offsets;
@@ -55,7 +55,7 @@ protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator alloc
 
   protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
     super(field, allocator);
-    this.offsets = new UInt4Vector(offsetsField, allocator);
+    this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
     this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
   }
 
@@ -110,8 +110,7 @@ public int getValueCapacity() {
   @Override
   protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
     return super.getMetadataBuilder()
-        .setGroupCount(getAccessor().getValueCount())
-        .setValueCount(getAccessor().getInnerValueCount())
+        .addChild(offsets.getMetadata())
         .addChild(vector.getMetadata());
   }
 
@@ -147,6 +146,21 @@ public DrillBuf[] getBuffers(boolean clear) {
     return buffers;
   }
 
+  @Override
+  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+    offsets.load(offsetMetadata, buffer);
+
+    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
+    if (getDataVector() == DEFAULT_DATA_VECTOR) {
+      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+    }
+
+    final int offsetLength = offsetMetadata.getBufferLength();
+    final int vectorLength = vectorMetadata.getBufferLength();
+    vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
+  }
+
   /**
    * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
    *

exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java

@@ -245,28 +245,28 @@ public Accessor getAccessor() {
 
   @Override
   public void load(SerializedField metadata, DrillBuf buf) {
-    List<SerializedField> fields = metadata.getChildList();
+    final List<SerializedField> fields = metadata.getChildList();
     valueCount = metadata.getValueCount();
 
     int bufOffset = 0;
-    for (SerializedField fmd : fields) {
-      MaterializedField fieldDef = MaterializedField.create(fmd);
+    for (final SerializedField child : fields) {
+      final MaterializedField fieldDef = MaterializedField.create(child);
 
-      ValueVector v = getChild(fieldDef.getLastName());
-      if (v == null) {
+      ValueVector vector = getChild(fieldDef.getLastName());
+      if (vector == null) {
         // if we arrive here, we didn't have a matching vector.
-        v = TypeHelper.getNewVector(fieldDef, allocator);
-        putChild(fieldDef.getLastName(), v);
+        vector = TypeHelper.getNewVector(fieldDef, allocator);
+        putChild(fieldDef.getLastName(), vector);
       }
-      if (fmd.getValueCount() == 0 && (!fmd.hasGroupCount() || fmd.getGroupCount() == 0)) {
-        v.clear();
+      if (child.getValueCount() == 0) {
+        vector.clear();
       } else {
-        v.load(fmd, buf.slice(bufOffset, fmd.getBufferLength()));
+        vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
       }
-      bufOffset += fmd.getBufferLength();
+      bufOffset += child.getBufferLength();
     }
 
-    Preconditions.checkArgument(bufOffset == buf.capacity());
+    assert bufOffset == buf.capacity();
   }
 
   @Override

exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java

@@ -17,8 +17,6 @@
  */
 package org.apache.drill.exec.vector.complex;
 
-import io.netty.buffer.DrillBuf;
-
 /**
  * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with
  * {@link RepeatedValueVector} subtypes.
@@ -39,18 +37,4 @@ public interface RepeatedFixedWidthVectorLike {
    * @param buf Incoming buffer.
    * @return The number of bytes of the buffer that were consumed.
    */
-  int load(int valueCount, int innerValueCount, DrillBuf buf);
-
-//  public interface RepeatedAccessor extends Accessor {
-//    public int getGroupCount();
-//    public int getValueCount();
-//    public int getGroupSizeAtIndex(int index);
-//    public ValueVector getAllChildValues();
-//  }
-//
-//  public interface RepeatedMutator extends Mutator {
-//    public void setValueCounts(int parentValueCount, int childValueCount);
-//    public void setRepetitionAtIndexSafe(int index, int repetitionCount);
-//    public BaseDataValueVector getDataVector();
-//  }
 }

exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java

@@ -18,6 +18,7 @@
 package org.apache.drill.exec.vector.complex;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 import io.netty.buffer.DrillBuf;
 
 import java.util.Collection;
@@ -186,11 +187,6 @@ public void allocateNew() throws OutOfMemoryRuntimeException {
       }
     }
 
-    @Override
-    protected SerializedField.Builder getMetadataBuilder() {
-      return super.getMetadataBuilder();
-    }
-
     @Override
     public TransferPair getTransferPair(FieldReference ref) {
       return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
@@ -216,23 +212,6 @@ public FieldReader getReader() {
       throw new UnsupportedOperationException();
     }
 
-    @Override
-    public void load(SerializedField metadata, DrillBuf buffer) {
-      //TODO(DRILL-2997): get rid of the notion of "group count" completely
-      final int valueCount = metadata.getGroupCount();
-      final int bufOffset = offsets.load(valueCount + 1, buffer);
-      final SerializedField childField = metadata.getChildList().get(0);
-      if (getDataVector() == DEFAULT_DATA_VECTOR) {
-        addOrGetVector(VectorDescriptor.create(childField.getMajorType()));
-      }
-
-      if (childField.getValueCount() == 0) {
-        vector.clear();
-      } else {
-        vector.load(childField, buffer.slice(bufOffset, childField.getBufferLength()));
-      }
-    }
-
     public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
       if(ephPair == null || ephPair.target != from) {
         ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
@@ -282,13 +261,13 @@ protected RepeatedListVector(MaterializedField field, BufferAllocator allocator,
     super(field, allocator, callBack);
     this.delegate = Preconditions.checkNotNull(delegate);
 
-    final Collection<MaterializedField> children = field.getChildren();
+    final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
     final int childSize = children.size();
-    // repeated list vector cannot have more than one child
-    assert childSize < 2;
-    final boolean hasChild = childSize == 1;
+    assert childSize < 3;
+    final boolean hasChild = childSize > 0;
     if (hasChild) {
-      final MaterializedField child = children.iterator().next();
+      // the last field is data field
+      final MaterializedField child = children.get(childSize-1);
       addOrGetVector(VectorDescriptor.create(child));
     }
   }
@@ -430,11 +409,6 @@ public void allocateNew(int valueCount, int innerValueCount) {
     getMutator().reset();
   }
 
-  @Override
-  public int load(int valueCount, int innerValueCount, DrillBuf buf) {
-    throw new UnsupportedOperationException();
-  }
-
   @Override
   public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
     if (name != null) {
@@ -443,15 +417,8 @@ public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
     return new VectorWithOrdinal(delegate.getDataVector(), 0);
   }
 
-
   public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
     delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
   }
 
-
-//  protected void setVector(ValueVector newVector) {
-//    vector = Preconditions.checkNotNull(newVector);
-//    getField().addChild(newVector.getField());
-//  }
-
 }