From a24efc90c7d8ba05fa532420830c48a9cbb9dcdf Mon Sep 17 00:00:00 2001 From: adeneche Date: Mon, 1 Aug 2016 15:28:08 -0700 Subject: [PATCH 1/3] [Java] UnionVector doesn't call allocateNew() when creating it's vectorType --- .../main/codegen/templates/UnionVector.java | 2 + .../arrow/vector/NonCleanBufferAllocator.java | 120 ++++++++++++++++++ .../apache/arrow/vector/TestValueVector.java | 42 ++++++ 3 files changed, 164 insertions(+) create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/NonCleanBufferAllocator.java diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 6042a5bf68352..98aaa0831e789 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -73,6 +73,8 @@ public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack this.allocator = allocator; this.internalMap = new MapVector("internal", allocator, callBack); this.typeVector = internalMap.addOrGet("types", new MajorType(MinorType.UINT1, DataMode.REQUIRED), UInt1Vector.class); + this.typeVector.allocateNew(); + this.typeVector.zeroVector(); this.field.addChild(internalMap.getField().clone()); this.majorType = field.getType(); this.callBack = callBack; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/NonCleanBufferAllocator.java b/java/vector/src/test/java/org/apache/arrow/vector/NonCleanBufferAllocator.java new file mode 100644 index 0000000000000..3c3b84aa12621 --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/NonCleanBufferAllocator.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import org.apache.arrow.memory.AllocationReservation; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.BufferManager; + +import io.netty.buffer.ArrowBuf; +import io.netty.buffer.ByteBufAllocator; + +/** + * Wrapper around a buffer delegate that populates any allocated buffer with a constant + * value. Useful for testing if value vectors are properly resetting their buffers. + */ +public class NonCleanBufferAllocator implements BufferAllocator { + + private final BufferAllocator delegate; + private final byte fillValue; + + NonCleanBufferAllocator(final BufferAllocator delegate, final byte fillValue) { + this.delegate = delegate; + this.fillValue = fillValue; + } + + @Override + public ArrowBuf buffer(int size) { + return buffer(size, null); + } + + @Override + public ArrowBuf buffer(int size, BufferManager manager) { + ArrowBuf buffer = delegate.buffer(size, manager); + // contaminate the buffer + for (int i = 0; i < buffer.capacity(); i++) { + buffer.setByte(i, fillValue); + } + + return buffer; + } + + @Override + public ByteBufAllocator getAsByteBufAllocator() { + return delegate.getAsByteBufAllocator(); + } + + @Override + public BufferAllocator newChildAllocator(String name, long initReservation, long maxAllocation) { + return delegate.newChildAllocator(name, initReservation, maxAllocation); + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public long getAllocatedMemory() { + return delegate.getAllocatedMemory(); + } + + @Override + public void setLimit(long newLimit) { + delegate.setLimit(newLimit); + } + + @Override + public long getLimit() { + return delegate.getLimit(); + } + + @Override + public long getPeakMemoryAllocation() { + return delegate.getPeakMemoryAllocation(); + } + + @Override + public AllocationReservation newReservation() { + return delegate.newReservation(); + } + + @Override + public ArrowBuf getEmpty() { + return delegate.getEmpty(); + } + + @Override + public String getName() { + return delegate.getName(); + } + + @Override + public boolean isOverLimit() { + return delegate.isOverLimit(); + } + + @Override + public String toVerboseString() { + return delegate.toVerboseString(); + } + + @Override + public void assertOpen() { + delegate.assertOpen(); + }} diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index ac3eebe98eab7..f00df0f7f40f4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.RepeatedListVector; import org.apache.arrow.vector.complex.RepeatedMapVector; +import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.types.MaterializedField; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; @@ -72,6 +73,47 @@ public void terminate() throws Exception { allocator.close(); } + @Test + public void testUnionVector() throws Exception { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); + + final BufferAllocator alloc = new NonCleanBufferAllocator(allocator, (byte) 100); + + UnionVector unionVector = new UnionVector(field, alloc, null); + + final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); + uInt4Holder.value = 100; + uInt4Holder.isSet = 1; + + try { + // write some data + final UnionVector.Mutator mutator = unionVector.getMutator(); + mutator.setType(0, MinorType.UINT4); + mutator.setSafe(0, uInt4Holder); + mutator.setType(2, MinorType.UINT4); + mutator.setSafe(2, uInt4Holder); + mutator.setValueCount(4); + + // check that what we wrote is correct + final UnionVector.Accessor accessor = unionVector.getAccessor(); + assertEquals(4, accessor.getValueCount()); + + assertEquals(false, accessor.isNull(0)); + assertEquals(100, accessor.getObject(0)); + + assertEquals(true, accessor.isNull(1)); + + assertEquals(false, accessor.isNull(2)); + assertEquals(100, accessor.getObject(2)); + + assertEquals(true, accessor.isNull(3)); + + } finally { + unionVector.clear(); + } + } + + @Test(expected = OversizedAllocationException.class) public void testFixedVectorReallocation() { final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); From 24b72f3dcb56ae7390a849baf553d095f0b77453 Mon Sep 17 00:00:00 2001 From: adeneche Date: Mon, 1 Aug 2016 16:02:15 -0700 Subject: [PATCH 2/3] added a separate TestUnionVector --- .../apache/arrow/vector/TestUnionVector.java | 88 +++++++++++++++++++ .../apache/arrow/vector/TestValueVector.java | 42 --------- 2 files changed, 88 insertions(+), 42 deletions(-) create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java new file mode 100644 index 0000000000000..f703e514e6f6c --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.junit.Assert.assertEquals; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.UnionVector; +import org.apache.arrow.vector.holders.NullableUInt4Holder; +import org.apache.arrow.vector.holders.UInt4Holder; +import org.apache.arrow.vector.types.MaterializedField; +import org.apache.arrow.vector.types.Types; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestUnionVector { + private final static String EMPTY_SCHEMA_PATH = ""; + + private BufferAllocator allocator; + + @Before + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void terminate() throws Exception { + allocator.close(); + } + + @Test + public void testUnionVector() throws Exception { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); + + final BufferAllocator alloc = new NonCleanBufferAllocator(allocator, (byte) 100); + + UnionVector unionVector = new UnionVector(field, alloc, null); + + final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); + uInt4Holder.value = 100; + uInt4Holder.isSet = 1; + + try { + // write some data + final UnionVector.Mutator mutator = unionVector.getMutator(); + mutator.setType(0, Types.MinorType.UINT4); + mutator.setSafe(0, uInt4Holder); + mutator.setType(2, Types.MinorType.UINT4); + mutator.setSafe(2, uInt4Holder); + mutator.setValueCount(4); + + // check that what we wrote is correct + final UnionVector.Accessor accessor = unionVector.getAccessor(); + assertEquals(4, accessor.getValueCount()); + + assertEquals(false, accessor.isNull(0)); + assertEquals(100, accessor.getObject(0)); + + assertEquals(true, accessor.isNull(1)); + + assertEquals(false, accessor.isNull(2)); + assertEquals(100, accessor.getObject(2)); + + assertEquals(true, accessor.isNull(3)); + + } finally { + unionVector.clear(); + } + } + +} diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index f00df0f7f40f4..ac3eebe98eab7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -28,7 +28,6 @@ import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.RepeatedListVector; import org.apache.arrow.vector.complex.RepeatedMapVector; -import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.types.MaterializedField; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; @@ -73,47 +72,6 @@ public void terminate() throws Exception { allocator.close(); } - @Test - public void testUnionVector() throws Exception { - final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); - - final BufferAllocator alloc = new NonCleanBufferAllocator(allocator, (byte) 100); - - UnionVector unionVector = new UnionVector(field, alloc, null); - - final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); - uInt4Holder.value = 100; - uInt4Holder.isSet = 1; - - try { - // write some data - final UnionVector.Mutator mutator = unionVector.getMutator(); - mutator.setType(0, MinorType.UINT4); - mutator.setSafe(0, uInt4Holder); - mutator.setType(2, MinorType.UINT4); - mutator.setSafe(2, uInt4Holder); - mutator.setValueCount(4); - - // check that what we wrote is correct - final UnionVector.Accessor accessor = unionVector.getAccessor(); - assertEquals(4, accessor.getValueCount()); - - assertEquals(false, accessor.isNull(0)); - assertEquals(100, accessor.getObject(0)); - - assertEquals(true, accessor.isNull(1)); - - assertEquals(false, accessor.isNull(2)); - assertEquals(100, accessor.getObject(2)); - - assertEquals(true, accessor.isNull(3)); - - } finally { - unionVector.clear(); - } - } - - @Test(expected = OversizedAllocationException.class) public void testFixedVectorReallocation() { final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); From e6cf83f6b25a11f65055e1e06b0abcbb03c94e47 Mon Sep 17 00:00:00 2001 From: adeneche Date: Tue, 2 Aug 2016 14:25:32 -0700 Subject: [PATCH 3/3] renamed UncleanBufferAllocator to DirtyBufferAllocator --- ...NonCleanBufferAllocator.java => DirtyBufferAllocator.java} | 4 ++-- .../test/java/org/apache/arrow/vector/TestUnionVector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename java/vector/src/test/java/org/apache/arrow/vector/{NonCleanBufferAllocator.java => DirtyBufferAllocator.java} (95%) diff --git a/java/vector/src/test/java/org/apache/arrow/vector/NonCleanBufferAllocator.java b/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java similarity index 95% rename from java/vector/src/test/java/org/apache/arrow/vector/NonCleanBufferAllocator.java rename to java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java index 3c3b84aa12621..cc6b9ec51d61c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/NonCleanBufferAllocator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/DirtyBufferAllocator.java @@ -28,12 +28,12 @@ * Wrapper around a buffer delegate that populates any allocated buffer with a constant * value. Useful for testing if value vectors are properly resetting their buffers. */ -public class NonCleanBufferAllocator implements BufferAllocator { +public class DirtyBufferAllocator implements BufferAllocator { private final BufferAllocator delegate; private final byte fillValue; - NonCleanBufferAllocator(final BufferAllocator delegate, final byte fillValue) { + DirtyBufferAllocator(final BufferAllocator delegate, final byte fillValue) { this.delegate = delegate; this.fillValue = fillValue; } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index f703e514e6f6c..8f19b3191ba15 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -49,7 +49,7 @@ public void terminate() throws Exception { public void testUnionVector() throws Exception { final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); - final BufferAllocator alloc = new NonCleanBufferAllocator(allocator, (byte) 100); + final BufferAllocator alloc = new DirtyBufferAllocator(allocator, (byte) 100); UnionVector unionVector = new UnionVector(field, alloc, null);