From 10023af3cdb3e476ce8f97ad93a949dc07c1eef8 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 27 Aug 2015 20:30:11 +0900 Subject: [PATCH 01/15] TAJO-1738: Improve off-heap RowBlock --- .../org/apache/tajo/storage/RowStoreUtil.java | 140 ++++---- .../org/apache/tajo/storage/BufferPool.java | 4 +- .../apache/tajo/tuple/BaseTupleBuilder.java | 64 ++-- .../org/apache/tajo/tuple/RowBlockReader.java | 6 +- .../org/apache/tajo/tuple/TupleBuilder.java | 2 +- .../tajo/tuple/memory}/DirectBufTuple.java | 13 +- .../tuple/memory}/FixedSizeLimitSpec.java | 2 +- .../apache/tajo/tuple/memory}/HeapTuple.java | 16 +- .../apache/tajo/tuple/memory/MemoryBlock.java | 143 ++++++++ .../tajo/tuple/memory/MemoryRowBlock.java | 155 +++++++++ .../tajo/tuple/memory/OffHeapMemoryBlock.java | 218 +++++++++++++ .../tuple/memory}/OffHeapRowBlockReader.java | 19 +- .../tuple/memory/OffHeapRowBlockUtils.java | 117 +++++++ .../tuple/memory}/OffHeapRowBlockWriter.java | 18 +- .../tajo/tuple/memory/OffHeapRowWriter.java | 305 ++++++++++++++++++ .../tuple/memory}/ResizableLimitSpec.java | 2 +- .../apache/tajo/tuple/memory/RowBlock.java | 48 +++ .../apache/tajo/tuple/memory}/RowWriter.java | 47 +-- .../tajo/tuple/memory}/UnSafeTuple.java | 175 +++++----- .../memory}/UnSafeTupleBytesComparator.java | 2 +- .../tajo/tuple/memory}/ZeroCopyTuple.java | 10 +- .../tajo/tuple/TestBaseTupleBuilder.java | 40 +-- .../tajo/tuple/memory}/TestHeapTuple.java | 13 +- .../tajo/tuple/memory/TestMemoryRowBlock.java | 135 ++++---- .../tajo/tuple/memory}/TestResizableSpec.java | 2 +- .../org/apache/tajo/storage/RowStoreUtil.java | 54 ---- .../tajo/tuple/offheap/OffHeapMemory.java | 102 ------ .../tajo/tuple/offheap/OffHeapRowBlock.java | 213 ------------ .../tuple/offheap/OffHeapRowBlockUtils.java | 54 ---- .../tajo/tuple/offheap/OffHeapRowWriter.java | 232 ------------- .../src/main/resources/storage-default.xml | 20 +- .../src/test/resources/storage-default.xml | 22 +- .../storage/rawfile/DirectRawFileScanner.java | 39 ++- .../storage/rawfile/DirectRawFileWriter.java | 92 ++---- .../tajo/storage/text/ByteBufLineReader.java | 4 +- .../org/apache/tajo/storage/TestStorages.java | 138 ++++---- .../tajo/storage/raw/TestDirectRawFile.java | 21 +- .../src/test/resources/storage-default.xml | 186 ----------- 38 files changed, 1511 insertions(+), 1362 deletions(-) rename {tajo-storage/tajo-storage-common => tajo-common}/src/main/java/org/apache/tajo/storage/BufferPool.java (97%) rename {tajo-storage/tajo-storage-common => tajo-common}/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java (50%) rename {tajo-storage/tajo-storage-common => tajo-common}/src/main/java/org/apache/tajo/tuple/RowBlockReader.java (94%) rename {tajo-storage/tajo-storage-common => tajo-common}/src/main/java/org/apache/tajo/tuple/TupleBuilder.java (95%) rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/DirectBufTuple.java (79%) rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/FixedSizeLimitSpec.java (96%) rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/HeapTuple.java (94%) create mode 100644 tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapMemoryBlock.java rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/OffHeapRowBlockReader.java (73%) create mode 100644 tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/OffHeapRowBlockWriter.java (75%) create mode 100644 tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/ResizableLimitSpec.java (99%) create mode 100644 tajo-common/src/main/java/org/apache/tajo/tuple/memory/RowBlock.java rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/RowWriter.java (63%) rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/UnSafeTuple.java (58%) rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/UnSafeTupleBytesComparator.java (98%) rename {tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap => tajo-common/src/main/java/org/apache/tajo/tuple/memory}/ZeroCopyTuple.java (78%) rename {tajo-storage/tajo-storage-common => tajo-common}/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java (59%) rename {tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap => tajo-common/src/test/java/org/apache/tajo/tuple/memory}/TestHeapTuple.java (72%) rename tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java => tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java (82%) rename {tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap => tajo-common/src/test/java/org/apache/tajo/tuple/memory}/TestResizableSpec.java (98%) delete mode 100644 tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java delete mode 100644 tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java delete mode 100644 tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java delete mode 100644 tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java delete mode 100644 tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml diff --git a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java index 4c7fe0a91e..bc23af8634 100644 --- a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java +++ b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java @@ -70,7 +70,7 @@ private RowStoreDecoder(Schema schema) { } - public Tuple toTuple(byte [] bytes) { + public Tuple toTuple(byte[] bytes) { nullFlags.clear(); ByteBuffer bb = ByteBuffer.wrap(bytes); Tuple tuple = new VTuple(schema.size()); @@ -81,7 +81,7 @@ public Tuple toTuple(byte [] bytes) { nullFlags.fromByteBuffer(bb); bb.limit(bytes.length); - for (int i =0; i < schema.size(); i++) { + for (int i = 0; i < schema.size(); i++) { if (nullFlags.get(i)) { tuple.put(i, DatumFactory.createNullDatum()); continue; @@ -90,73 +90,75 @@ public Tuple toTuple(byte [] bytes) { col = schema.getColumn(i); type = col.getDataType(); switch (type.getType()) { - case BOOLEAN: tuple.put(i, DatumFactory.createBool(bb.get())); break; - case BIT: - byte b = bb.get(); - tuple.put(i, DatumFactory.createBit(b)); - break; - - case CHAR: - byte [] _str = new byte[type.getLength()]; - bb.get(_str); - tuple.put(i, DatumFactory.createChar(_str)); - break; - - case INT2: - short s = bb.getShort(); - tuple.put(i, DatumFactory.createInt2(s)); - break; - - case INT4: - case DATE: - int i_ = bb.getInt(); - tuple.put(i, DatumFactory.createFromInt4(type, i_)); - break; - - case INT8: - case TIME: - case TIMESTAMP: - long l = bb.getLong(); - tuple.put(i, DatumFactory.createFromInt8(type, l)); - break; - - case INTERVAL: - int month = bb.getInt(); - long milliseconds = bb.getLong(); - tuple.put(i, new IntervalDatum(month, milliseconds)); - break; - - case FLOAT4: - float f = bb.getFloat(); - tuple.put(i, DatumFactory.createFloat4(f)); - break; - - case FLOAT8: - double d = bb.getDouble(); - tuple.put(i, DatumFactory.createFloat8(d)); - break; - - case TEXT: - byte [] _string = new byte[bb.getInt()]; - bb.get(_string); - tuple.put(i, DatumFactory.createText(_string)); - break; - - case BLOB: - byte [] _bytes = new byte[bb.getInt()]; - bb.get(_bytes); - tuple.put(i, DatumFactory.createBlob(_bytes)); - break; - - case INET4: - byte [] _ipv4 = new byte[4]; - bb.get(_ipv4); - tuple.put(i, DatumFactory.createInet4(_ipv4)); - break; - - default: - throw new TajoRuntimeException( - new UnsupportedException("data type '" + col.getDataType().getType().name() + "'")); + case BOOLEAN: + tuple.put(i, DatumFactory.createBool(bb.get())); + break; + case BIT: + byte b = bb.get(); + tuple.put(i, DatumFactory.createBit(b)); + break; + + case CHAR: + byte[] _str = new byte[type.getLength()]; + bb.get(_str); + tuple.put(i, DatumFactory.createChar(_str)); + break; + + case INT2: + short s = bb.getShort(); + tuple.put(i, DatumFactory.createInt2(s)); + break; + + case INT4: + case DATE: + int i_ = bb.getInt(); + tuple.put(i, DatumFactory.createFromInt4(type, i_)); + break; + + case INT8: + case TIME: + case TIMESTAMP: + long l = bb.getLong(); + tuple.put(i, DatumFactory.createFromInt8(type, l)); + break; + + case INTERVAL: + int month = bb.getInt(); + long milliseconds = bb.getLong(); + tuple.put(i, new IntervalDatum(month, milliseconds)); + break; + + case FLOAT4: + float f = bb.getFloat(); + tuple.put(i, DatumFactory.createFloat4(f)); + break; + + case FLOAT8: + double d = bb.getDouble(); + tuple.put(i, DatumFactory.createFloat8(d)); + break; + + case TEXT: + byte[] _string = new byte[bb.getInt()]; + bb.get(_string); + tuple.put(i, DatumFactory.createText(_string)); + break; + + case BLOB: + byte[] _bytes = new byte[bb.getInt()]; + bb.get(_bytes); + tuple.put(i, DatumFactory.createBlob(_bytes)); + break; + + case INET4: + byte[] _ipv4 = new byte[4]; + bb.get(_ipv4); + tuple.put(i, DatumFactory.createInet4(_ipv4)); + break; + + default: + throw new TajoRuntimeException( + new UnsupportedException("data type '" + col.getDataType().getType().name() + "'")); } } return tuple; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BufferPool.java b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java similarity index 97% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BufferPool.java rename to tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java index d611ee378f..be298fc0f8 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BufferPool.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java @@ -119,7 +119,7 @@ public static void forceRelease(ByteBuf buf) { * @param buf * @param minWritableBytes required minimum writable size */ - public static void ensureWritable(ByteBuf buf, int minWritableBytes) { - buf.ensureWritable(minWritableBytes); + public static ByteBuf ensureWritable(ByteBuf buf, int minWritableBytes) { + return buf.ensureWritable(minWritableBytes); } } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java b/tajo-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java similarity index 50% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java index c1835dff98..4de4dd22b5 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java @@ -18,73 +18,53 @@ package org.apache.tajo.tuple; +import io.netty.util.internal.PlatformDependent; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaUtil; +import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.tuple.offheap.HeapTuple; -import org.apache.tajo.tuple.offheap.OffHeapRowWriter; -import org.apache.tajo.tuple.offheap.ZeroCopyTuple; +import org.apache.tajo.tuple.memory.*; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.Deallocatable; -import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.UnsafeUtil; -import sun.misc.Unsafe; -import sun.nio.ch.DirectBuffer; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; public class BaseTupleBuilder extends OffHeapRowWriter implements TupleBuilder, Deallocatable { private static final Log LOG = LogFactory.getLog(BaseTupleBuilder.class); - private static final Unsafe UNSAFE = UnsafeUtil.unsafe; - // buffer - private ByteBuffer buffer; - private long address; + private MemoryBlock memoryBlock; - public BaseTupleBuilder(Schema schema) { - super(SchemaUtil.toDataTypes(schema)); - buffer = ByteBuffer.allocateDirect(64 * StorageUnit.KB).order(ByteOrder.nativeOrder()); - address = UnsafeUtil.getAddress(buffer); + public BaseTupleBuilder(DataType[] schema) { + super(schema); + this.memoryBlock = new OffHeapMemoryBlock(new ResizableLimitSpec(64 * StorageUnit.KB)); } @Override public long address() { - return address; + return memoryBlock.address(); } public void ensureSize(int size) { - if (buffer.remaining() - size < 0) { // check the remain size - // enlarge new buffer and copy writing data - int newBlockSize = UnsafeUtil.alignedSize(buffer.capacity() * 2); - ByteBuffer newByteBuf = ByteBuffer.allocateDirect(newBlockSize); - long newAddress = ((DirectBuffer)newByteBuf).address(); - UNSAFE.copyMemory(this.address, newAddress, buffer.limit()); - LOG.debug("Increase the buffer size to " + FileUtil.humanReadableByteCount(newBlockSize, false)); - - // release existing buffer and replace variables - UnsafeUtil.free(buffer); - buffer = newByteBuf; - address = newAddress; - } + memoryBlock.ensureSize(size); } @Override public int position() { - return 0; + return memoryBlock.writerPosition(); } @Override public void forward(int length) { + memoryBlock.writerPosition(memoryBlock.writerPosition() + length); + } + + @Override + public boolean startRow() { + return super.startRow(); } @Override public void endRow() { super.endRow(); - buffer.position(0).limit(offset()); } @Override @@ -93,20 +73,20 @@ public Tuple build() { } public HeapTuple buildToHeapTuple() { - byte [] bytes = new byte[buffer.limit()]; - UNSAFE.copyMemory(null, address, bytes, UnsafeUtil.ARRAY_BOOLEAN_BASE_OFFSET, buffer.limit()); + byte[] bytes = new byte[memoryBlock.readableBytes()]; + PlatformDependent.copyMemory(memoryBlock.address(), bytes, memoryBlock.readerPosition(), bytes.length); + memoryBlock.writerPosition(0); return new HeapTuple(bytes, dataTypes()); } public ZeroCopyTuple buildToZeroCopyTuple() { ZeroCopyTuple zcTuple = new ZeroCopyTuple(); - zcTuple.set(buffer, 0, buffer.limit(), dataTypes()); + zcTuple.set(memoryBlock, memoryBlock.readerPosition(), memoryBlock.readableBytes(), dataTypes()); + memoryBlock.writerPosition(0); return zcTuple; } public void release() { - UnsafeUtil.free(buffer); - buffer = null; - address = 0; + memoryBlock.release(); } } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java b/tajo-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java similarity index 94% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java index be734e1e29..59cdef577f 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java @@ -27,7 +27,9 @@ public interface RowBlockReader { * * @return True if tuple block is filled with tuples. Otherwise, It will return false. */ - public boolean next(T tuple); + boolean next(T tuple); - public void reset(); + void reset(); + + long remainForRead(); } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java b/tajo-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java similarity index 95% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java index c43c018ada..5b4bd80241 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java @@ -19,7 +19,7 @@ package org.apache.tajo.tuple; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.tuple.offheap.RowWriter; +import org.apache.tajo.tuple.memory.RowWriter; public interface TupleBuilder extends RowWriter { public Tuple build(); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java similarity index 79% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java index 9662d5a867..4d5b0a93f0 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java @@ -16,10 +16,9 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import org.apache.tajo.util.Deallocatable; -import org.apache.tajo.util.UnsafeUtil; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -27,15 +26,17 @@ import static org.apache.tajo.common.TajoDataTypes.DataType; public class DirectBufTuple extends UnSafeTuple implements Deallocatable { - private ByteBuffer bb; + private MemoryBlock memoryBlock; public DirectBufTuple(int length, DataType[] types) { - bb = ByteBuffer.allocateDirect(length).order(ByteOrder.nativeOrder()); - set(bb, 0, length, types); + ByteBuffer bb = ByteBuffer.allocateDirect(length).order(ByteOrder.nativeOrder()); + memoryBlock = new OffHeapMemoryBlock(bb, new FixedSizeLimitSpec(length)); + + set(memoryBlock, 0, length, types); } @Override public void release() { - UnsafeUtil.free(bb); + memoryBlock.release(); } } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/FixedSizeLimitSpec.java similarity index 96% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/FixedSizeLimitSpec.java index a327123ec3..367d90d2bf 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/FixedSizeLimitSpec.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; /** * Fixed size limit specification diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java similarity index 94% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java index 9b69536c0f..c608a6d86a 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; @@ -37,7 +37,7 @@ import static org.apache.tajo.common.TajoDataTypes.DataType; -public class HeapTuple implements Tuple { +public class HeapTuple implements Tuple, Cloneable { private static final Unsafe UNSAFE = UnsafeUtil.unsafe; private static final long BASE_OFFSET = UnsafeUtil.ARRAY_BYTE_BASE_OFFSET; @@ -73,12 +73,12 @@ public ByteBuffer nioBuffer() { } private int getFieldOffset(int fieldId) { - return UNSAFE.getInt(data, BASE_OFFSET + SizeOf.SIZE_OF_INT + (fieldId * SizeOf.SIZE_OF_INT)); + return UNSAFE.getInt(data, BASE_OFFSET + (SizeOf.SIZE_OF_INT + (fieldId * SizeOf.SIZE_OF_INT))); } private int checkNullAndGetOffset(int fieldId) { int offset = getFieldOffset(fieldId); - if (offset == OffHeapRowBlock.NULL_FIELD_OFFSET) { + if (offset == MemoryRowBlock.NULL_FIELD_OFFSET) { throw new RuntimeException("Invalid Field Access: " + fieldId); } return offset; @@ -86,17 +86,17 @@ private int checkNullAndGetOffset(int fieldId) { @Override public boolean contains(int fieldid) { - return getFieldOffset(fieldid) > OffHeapRowBlock.NULL_FIELD_OFFSET; + return getFieldOffset(fieldid) > MemoryRowBlock.NULL_FIELD_OFFSET; } @Override public boolean isBlank(int fieldid) { - return getFieldOffset(fieldid) == OffHeapRowBlock.NULL_FIELD_OFFSET; + return getFieldOffset(fieldid) == MemoryRowBlock.NULL_FIELD_OFFSET; } @Override public boolean isBlankOrNull(int fieldid) { - return getFieldOffset(fieldid) == OffHeapRowBlock.NULL_FIELD_OFFSET; + return getFieldOffset(fieldid) == MemoryRowBlock.NULL_FIELD_OFFSET; } @Override @@ -269,7 +269,7 @@ public char[] getUnicodeChars(int fieldId) { @Override public Tuple clone() throws CloneNotSupportedException { - return this; + return (HeapTuple) super.clone(); } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java new file mode 100644 index 0000000000..918ee472ac --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.tuple.memory; + +import org.apache.tajo.util.Deallocatable; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.GatheringByteChannel; +import java.nio.channels.ScatteringByteChannel; + +/** + * This interface provides a memory spec for off-heap or heap + */ +public interface MemoryBlock extends Deallocatable { + + /** + * Current memory address of this buffer + * + * @return the memory address + */ + long address(); + + /** + * @return if true, the buffer has a reference to the low-level memory address + */ + boolean hasAddress(); + + /** + * @return the number of bytes this buffer can contain. + */ + int capacity(); + + /** + * reset the buffer + */ + void clear(); + + /** + * @return true if this buffer has an remaining bytes. + */ + boolean isReadable(); + + /** + * @return the number of readable bytes in this buffer + */ + int readableBytes(); + + /** + * @return the current reader position of this buffer + */ + int readerPosition(); + + /** + * Sets the reader position of this buffer + */ + void readerPosition(int pos); + + /** + * @return true if this buffer is not full filled + */ + boolean isWritable(); + + /** + * @return the number of writable bytes in this buffer + */ + int writableBytes(); + + /** + * @return the current writer position of this buffer + */ + int writerPosition(); + + /** + * Sets the writer position of this buffer + */ + void writerPosition(int pos); + + /** + * Ensure that this buffer has enough remaining space to add the capacity. + * Creates and copies to a new buffer if necessary + * + * @param size Size to add + */ + void ensureSize(int size); + + /** + * Transfers the content of the channel to this buffer + * @param in the input channel + * @return the actual number of bytes read in channel + */ + int writeBytes(ScatteringByteChannel in) throws IOException; + + /** + * Transfers the content of this bufer to the byte array + * @param dst the destination byte array + * @param dstIndex the first index of the destination + * @param length the number of bytes to transfer + * @return the actual number of bytes transfers to the destination byte array + */ + int getBytes(byte[] dst, int dstIndex, int length) throws IOException; + + /** + * Transfers the content of this buffer to the channel + * @param out the output channel + * @param length the maximum number of bytes to transfer + * @return the actual number of bytes transfers to the channel + */ + int writeTo(GatheringByteChannel out, int length) throws IOException; + + int writeTo(GatheringByteChannel out) throws IOException; + + /** + * Transfers the content of this buffer to the stream + * @param out the output stream + * @param length the maximum number of bytes to transfer + * @return the actual number of bytes transfers to the stream + */ + int writeTo(OutputStream out, int length) throws IOException; + + int writeTo(OutputStream out) throws IOException; + + /** + * @return a {@MemoryBlock} which shares the whole region of this. + */ + MemoryBlock duplicate(); +} diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java new file mode 100644 index 0000000000..493a2d8d14 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java @@ -0,0 +1,155 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.tuple.memory; + +import com.google.common.annotations.VisibleForTesting; +import io.netty.util.internal.PlatformDependent; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.tuple.RowBlockReader; +import org.apache.tajo.util.Deallocatable; +import org.apache.tajo.util.SizeOf; +import org.apache.tajo.util.TUtil; + +import java.io.IOException; +import java.nio.channels.ScatteringByteChannel; + +import static org.apache.tajo.common.TajoDataTypes.DataType; + +public class MemoryRowBlock implements RowBlock, Deallocatable { + public static final int NULL_FIELD_OFFSET = -1; + + private DataType[] dataTypes; + + // Basic States + private int maxRowNum = Integer.MAX_VALUE; // optional + private int rowNum; + + private RowWriter builder; + private MemoryBlock memory; + + public MemoryRowBlock(DataType[] dataTypes, ResizableLimitSpec limitSpec, boolean offheap) { + if (offheap) { + this.memory = new OffHeapMemoryBlock(limitSpec); + } else { + throw new TajoRuntimeException(new NotImplementedException("Heap memory not implemented yet")); + } + this.dataTypes = dataTypes; + this.builder = new OffHeapRowBlockWriter(this); + } + + public MemoryRowBlock(MemoryRowBlock rowBlock) { + this.memory = TUtil.checkTypeAndGet(rowBlock.getMemory().duplicate(), OffHeapMemoryBlock.class); + this.rowNum = rowBlock.rowNum; + this.dataTypes = rowBlock.dataTypes; + this.builder = new OffHeapRowBlockWriter(this); + } + + @VisibleForTesting + public MemoryRowBlock(DataType[] dataTypes, int bytes) { + this(dataTypes, new ResizableLimitSpec(bytes), true); + } + + + @Override + public void clear() { + reset(); + memory.clear(); + } + + private void reset() { + rowNum = 0; + builder.clear(); + } + + @Override + public int capacity() { + return memory.capacity(); + } + + public int maxRowNum() { + return maxRowNum; + } + + @Override + public int rows() { + return rowNum; + } + + @Override + public void setRows(int rowNum) { + this.rowNum = rowNum; + } + + @Override + public DataType[] getDataTypes() { + return dataTypes; + } + + @Override + public boolean copyFromChannel(ScatteringByteChannel channel) + throws IOException { + reset(); + + int readBytes = memory.writeBytes(channel); + + if (readBytes > 0) { + // get row capacity in buffer + while (memory.isReadable()) { + if (memory.readableBytes() < SizeOf.SIZE_OF_INT) { + return true; + } + + int recordSize = PlatformDependent.getInt(memory.address() + memory.readerPosition()); + assert recordSize > 0; + if (memory.readableBytes() < recordSize) { + return true; + } else { + memory.readerPosition(memory.readerPosition() + recordSize); + } + + rowNum++; + } + + return true; + } else { + return false; + } + } + + @Override + public RowWriter getWriter() { + return builder; + } + + @Override + public MemoryBlock getMemory() { + return memory; + } + + @Override + public void release() { + memory.release(); + } + + @Override + public RowBlockReader getReader() { + return new OffHeapRowBlockReader(this); + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapMemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapMemoryBlock.java new file mode 100644 index 0000000000..ac035ed352 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapMemoryBlock.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.tuple.memory; + +import com.google.common.base.Preconditions; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.storage.BufferPool; +import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.UnsafeUtil; +import sun.misc.Unsafe; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.GatheringByteChannel; +import java.nio.channels.ScatteringByteChannel; + +public class OffHeapMemoryBlock implements MemoryBlock { + private static final Log LOG = LogFactory.getLog(OffHeapMemoryBlock.class); + + protected ByteBuf buffer; + protected ResizableLimitSpec limitSpec; + protected static final Unsafe UNSAFE = UnsafeUtil.unsafe; + + protected OffHeapMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { + Preconditions.checkState(buffer.hasMemoryAddress()); + this.buffer = buffer; + this.limitSpec = limitSpec; + } + + protected OffHeapMemoryBlock(ByteBuffer buffer, ResizableLimitSpec limitSpec) { + Preconditions.checkState(buffer.isDirect()); + this.buffer = Unpooled.wrappedBuffer(buffer); + this.limitSpec = limitSpec; + } + + public OffHeapMemoryBlock(ResizableLimitSpec limitSpec) { + this(BufferPool.directBuffer((int) limitSpec.initialSize(), + (int) limitSpec.limit()).order(ByteOrder.nativeOrder()), limitSpec); + } + + @Override + public long address() { + return buffer.memoryAddress(); + } + + @Override + public boolean hasAddress() { + return true; + } + + @Override + public int capacity() { + return buffer.capacity(); + } + + @Override + public void clear() { + buffer.clear(); + } + + @Override + public boolean isReadable() { + return buffer.isReadable(); + } + + @Override + public int readableBytes() { + return buffer.readableBytes(); + } + + @Override + public int readerPosition() { + return buffer.readerIndex(); + } + + @Override + public void readerPosition(int pos) { + buffer.readerIndex(pos); + } + + @Override + public boolean isWritable() { + return buffer.isWritable(); + } + + @Override + public int writableBytes() { + return buffer.writableBytes(); + } + + @Override + public void writerPosition(int pos) { + buffer.writerIndex(pos); + } + + @Override + public int writerPosition() { + return buffer.writerIndex(); + } + + + @Override + public void ensureSize(int size) { + if (!buffer.isWritable(size)) { + if (!limitSpec.canIncrease(buffer.capacity())) { + throw new RuntimeException("Cannot increase RowBlock anymore."); + } + + int newBlockSize = limitSpec.increasedSize(buffer.capacity()); + resize(newBlockSize); + LOG.info("Increase DirectRowBlock to " + FileUtil.humanReadableByteCount(newBlockSize, false)); + } + } + + private void resize(int newSize) { + Preconditions.checkArgument(newSize > 0, "Size must be greater than 0 bytes"); + + if (newSize > limitSpec.limit()) { + throw new RuntimeException("Resize cannot exceed the capacity limit"); + } + + if (newSize < buffer.capacity()) { + LOG.warn("The capacity reduction is ignored."); + } + + int newBlockSize = UnsafeUtil.alignedSize(newSize); + buffer = BufferPool.ensureWritable(buffer, newBlockSize); + } + + @Override + public void release() { + buffer.release(); + this.buffer = null; + } + + @Override + public MemoryBlock duplicate() { + return new OffHeapMemoryBlock(buffer.duplicate().readerIndex(0), limitSpec); + } + + @Override + public int writeBytes(ScatteringByteChannel channel) throws IOException { + + if (buffer.readableBytes() > 0) { + this.buffer.markReaderIndex(); + this.buffer.discardReadBytes(); // compact the buffer + } else { + buffer.clear(); + } + + int readBytes = 0; + while (buffer.writableBytes() > 0) { + int localReadBytes = buffer.writeBytes(channel, buffer.writableBytes()); + if (localReadBytes < 0) { + break; + } + readBytes += localReadBytes; + } + + return readBytes; + } + + @Override + public int getBytes(byte[] bytes, int dstIndex, int length) throws IOException { + int readableBytes = buffer.readableBytes(); + buffer.readBytes(bytes, dstIndex, length); + return readableBytes - buffer.readableBytes(); + } + + @Override + public int writeTo(GatheringByteChannel channel, int length) throws IOException { + return buffer.readBytes(channel, length); + } + + @Override + public int writeTo(GatheringByteChannel channel) throws IOException { + return buffer.readBytes(channel, buffer.readableBytes()); + } + + @Override + public int writeTo(OutputStream outputStream, int length) throws IOException { + buffer.readBytes(outputStream, length); + return length; + } + + @Override + public int writeTo(OutputStream outputStream) throws IOException { + int readableBytes = buffer.readableBytes(); + buffer.readBytes(outputStream, readableBytes); + return readableBytes - buffer.readableBytes(); + } + + @Override + public String toString() { + return "memory=" + FileUtil.humanReadableByteCount(capacity(), false) + "," + limitSpec; + } +} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockReader.java similarity index 73% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockReader.java index 4a9313f1b0..16f5b492ea 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockReader.java @@ -16,38 +16,37 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; +import io.netty.util.internal.PlatformDependent; import org.apache.tajo.tuple.RowBlockReader; -import org.apache.tajo.util.UnsafeUtil; -import sun.misc.Unsafe; public class OffHeapRowBlockReader implements RowBlockReader { - private static final Unsafe UNSAFE = UnsafeUtil.unsafe; - OffHeapRowBlock rowBlock; + private MemoryRowBlock rowBlock; // Read States private int curRowIdxForRead; private int curPosForRead; - public OffHeapRowBlockReader(OffHeapRowBlock rowBlock) { + public OffHeapRowBlockReader(MemoryRowBlock rowBlock) { this.rowBlock = rowBlock; } public long remainForRead() { - return rowBlock.memorySize - curPosForRead; + return rowBlock.capacity() - curPosForRead; } @Override public boolean next(ZeroCopyTuple tuple) { if (curRowIdxForRead < rowBlock.rows()) { - long recordStartPtr = rowBlock.address() + curPosForRead; - int recordLen = UNSAFE.getInt(recordStartPtr); - tuple.set(rowBlock.buffer, curPosForRead, recordLen, rowBlock.dataTypes); + long recordStartPtr = rowBlock.getMemory().address() + curPosForRead; + int recordLen = PlatformDependent.getInt(recordStartPtr); + tuple.set(rowBlock.getMemory(), curPosForRead, recordLen, rowBlock.getDataTypes()); curPosForRead += recordLen; curRowIdxForRead++; + rowBlock.getMemory().readerPosition(curPosForRead); return true; } else { diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java new file mode 100644 index 0000000000..f2f71738d0 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.tuple.memory; + +import com.google.common.collect.Lists; +import org.apache.tajo.datum.IntervalDatum; +import org.apache.tajo.datum.ProtobufDatum; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.storage.Tuple; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class OffHeapRowBlockUtils { + + public static List sort(MemoryRowBlock rowBlock, Comparator comparator) { + List tupleList = Lists.newArrayList(); + ZeroCopyTuple zcTuple = new ZeroCopyTuple(); + OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); + while(reader.next(zcTuple)) { + tupleList.add(zcTuple); + zcTuple = new ZeroCopyTuple(); + } + Collections.sort(tupleList, comparator); + return tupleList; + } + + public static Tuple[] sortToArray(MemoryRowBlock rowBlock, Comparator comparator) { + Tuple[] tuples = new Tuple[rowBlock.rows()]; + ZeroCopyTuple zcTuple = new ZeroCopyTuple(); + OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); + for (int i = 0; i < rowBlock.rows() && reader.next(zcTuple); i++) { + tuples[i] = zcTuple; + zcTuple = new ZeroCopyTuple(); + } + Arrays.sort(tuples, comparator); + return tuples; + } + + public static void convert(Tuple tuple, RowWriter writer) { + writer.startRow(); + + for (int i = 0; i < writer.dataTypes().length; i++) { + if (tuple.isBlankOrNull(i)) { + writer.skipField(); + continue; + } + switch (writer.dataTypes()[i].getType()) { + case BOOLEAN: + writer.putBool(tuple.getBool(i)); + break; + case BIT: + writer.putByte(tuple.getByte(i)); + break; + case INT1: + case INT2: + writer.putInt2(tuple.getInt2(i)); + break; + case INT4: + case DATE: + case INET4: + writer.putInt4(tuple.getInt4(i)); + break; + case INT8: + case TIMESTAMP: + case TIME: + writer.putInt8(tuple.getInt8(i)); + break; + case FLOAT4: + writer.putFloat4(tuple.getFloat4(i)); + break; + case FLOAT8: + writer.putFloat8(tuple.getFloat8(i)); + break; + case CHAR: + case TEXT: + writer.putText(tuple.getBytes(i)); + break; + case BLOB: + writer.putBlob(tuple.getBytes(i)); + break; + case INTERVAL: + writer.putInterval((IntervalDatum) tuple.getInterval(i)); + break; + case PROTOBUF: + writer.putProtoDatum((ProtobufDatum) tuple.getProtobufDatum(i)); + break; + case NULL_TYPE: + writer.skipField(); + break; + default: + throw new TajoRuntimeException( + new UnsupportedException("unknown data type '" + writer.dataTypes()[i].getType().name() + "'")); + } + } + writer.endRow(); + } +} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockWriter.java similarity index 75% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockWriter.java index d177e0caff..322d7ca43f 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockWriter.java @@ -16,33 +16,33 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import org.apache.tajo.common.TajoDataTypes; public class OffHeapRowBlockWriter extends OffHeapRowWriter { - OffHeapRowBlock rowBlock; + private RowBlock rowBlock; - OffHeapRowBlockWriter(OffHeapRowBlock rowBlock) { - super(rowBlock.dataTypes); + OffHeapRowBlockWriter(RowBlock rowBlock) { + super(rowBlock.getDataTypes()); this.rowBlock = rowBlock; } public long address() { - return rowBlock.address(); + return rowBlock.getMemory().address(); } public int position() { - return rowBlock.position(); + return rowBlock.getMemory().writerPosition(); } @Override public void forward(int length) { - rowBlock.position(position() + length); + rowBlock.getMemory().writerPosition(rowBlock.getMemory().writerPosition() + length); } public void ensureSize(int size) { - rowBlock.ensureSize(size); + rowBlock.getMemory().ensureSize(size); } @Override @@ -53,6 +53,6 @@ public void endRow() { @Override public TajoDataTypes.DataType[] dataTypes() { - return rowBlock.dataTypes; + return rowBlock.getDataTypes(); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java new file mode 100644 index 0000000000..8e50f888df --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java @@ -0,0 +1,305 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.tuple.memory; + +import io.netty.util.internal.PlatformDependent; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.datum.IntervalDatum; +import org.apache.tajo.datum.ProtobufDatum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.SizeOf; +import org.apache.tajo.util.TUtil; + +/** + * + * Row Record Structure + * + * | row length (4 bytes) | field 1 offset | field 2 offset | ... | field N offset| field 1 | field 2| ... | field N | + * 4 bytes 4 bytes 4 bytes + * + */ +public abstract class OffHeapRowWriter implements RowWriter { + /** record capacity + offset list */ + private final int headerSize; + + private final DataType[] dataTypes; + + private int curFieldIdx; + private int curFieldOffset; + private int curOffset; + + public OffHeapRowWriter(final DataType[] dataTypes) { + this.dataTypes = dataTypes; + this.headerSize = SizeOf.SIZE_OF_INT * (dataTypes.length + 1); + this.curFieldOffset = SizeOf.SIZE_OF_INT; + } + + /** + * Current memory address of the row + * + * @return The memory address + */ + public long recordStartAddr() { + return currentAddr() - curOffset; + } + + /** + * Memory address that point to the first byte of the buffer + * + * @return The memory address + */ + private long currentAddr() { + return address() + position(); + } + + /** + * Current memory address of the buffer + * + * @return The memory address + */ + public abstract long address(); + + public abstract void ensureSize(int size); + + public int offset() { + return position(); + } + + /** + * Current position + * + * @return The position + */ + public abstract int position(); + + /** + * Forward the address; + * + * @param length Length to be forwarded + */ + public abstract void forward(int length); + + + @Override + public void clear() { + curOffset = 0; + curFieldIdx = 0; + curFieldOffset = SizeOf.SIZE_OF_INT; + } + + @Override + public DataType[] dataTypes() { + return dataTypes; + } + + @Override + public boolean startRow() { + ensureSize(headerSize); + curOffset = headerSize; + curFieldOffset = SizeOf.SIZE_OF_INT; + curFieldIdx = 0; + forward(headerSize); + return true; + } + + @Override + public void endRow() { + long rowHeaderPos = recordStartAddr(); + // curOffset is equivalent to a byte length of this row. + OffHeapMemoryBlock.UNSAFE.putInt(rowHeaderPos, curOffset); + + //forward (record offset + fields offset) + rowHeaderPos += SizeOf.SIZE_OF_INT + curFieldOffset; + // set remain header field length + for (int i = curFieldIdx; i < dataTypes.length; i++) { + OffHeapMemoryBlock.UNSAFE.putInt(rowHeaderPos, MemoryRowBlock.NULL_FIELD_OFFSET); + rowHeaderPos += SizeOf.SIZE_OF_INT; + } + } + + @Override + public void skipField() { + // set header field length + putFieldHeader(currentAddr(), MemoryRowBlock.NULL_FIELD_OFFSET); + } + + /** + * set current buffer position and forward field length + * @param fieldLength + */ + private void forwardField(int fieldLength) { + forward(fieldLength); + curOffset += fieldLength; + + } + + private void putFieldHeader(long currentAddr, int length) { + long currentHeaderAddr = currentAddr - curOffset + curFieldOffset; + + // set header field length + OffHeapMemoryBlock.UNSAFE.putInt(currentHeaderAddr, length); + curFieldOffset += SizeOf.SIZE_OF_INT; + curFieldIdx++; + } + + @Override + public void putByte(byte val) { + ensureSize(SizeOf.SIZE_OF_BYTE); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putByte(addr, val); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_BYTE); + } + + @Override + public void putBool(boolean val) { + ensureSize(SizeOf.SIZE_OF_BOOL); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putByte(addr, (byte) (val ? 0x01 : 0x00)); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_BOOL); + } + + @Override + public void putInt2(short val) { + ensureSize(SizeOf.SIZE_OF_SHORT); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putShort(addr, val); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_SHORT); + } + + @Override + public void putInt4(int val) { + ensureSize(SizeOf.SIZE_OF_INT); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putInt(addr, val); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_INT); + } + + @Override + public void putInt8(long val) { + ensureSize(SizeOf.SIZE_OF_LONG); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putLong(addr, val); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_LONG); + } + + @Override + public void putFloat4(float val) { + ensureSize(SizeOf.SIZE_OF_INT); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putInt(addr, Float.floatToRawIntBits(val)); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_INT); + } + + @Override + public void putFloat8(double val) { + ensureSize(SizeOf.SIZE_OF_LONG); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putLong(addr, Double.doubleToRawLongBits(val)); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_LONG); + } + + @Override + public void putText(String val) { + byte[] bytes = val.getBytes(TextDatum.DEFAULT_CHARSET); + putText(bytes); + } + + @Override + public void putText(byte[] val) { + putBlob(val); + } + + @Override + public void putBlob(byte[] val) { + int bytesLen = val.length; + int fieldLen = SizeOf.SIZE_OF_INT + bytesLen; + + ensureSize(fieldLen); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putInt(addr, bytesLen); + PlatformDependent.copyMemory(val, 0, addr + SizeOf.SIZE_OF_INT, bytesLen); + putFieldHeader(addr, curOffset); + forwardField(fieldLen); + } + + @Override + public void putTimestamp(long val) { + putInt8(val); + } + + @Override + public void putDate(int val) { + putInt4(val); + } + + @Override + public void putTime(long val) { + putInt8(val); + } + + @Override + public void putInterval(IntervalDatum val) { + ensureSize(SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_LONG); + long addr = currentAddr(); + + OffHeapMemoryBlock.UNSAFE.putInt(addr, val.getMonths()); + OffHeapMemoryBlock.UNSAFE.putLong(addr + SizeOf.SIZE_OF_INT, val.getMilliSeconds()); + putFieldHeader(addr, curOffset); + forwardField(SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_LONG); + } + + @Override + public void putInet4(int val) { + putInt4(val); + } + + @Override + public void putProtoDatum(ProtobufDatum val) { + putBlob(val.asByteArray()); + } + + @Override + public void addTuple(Tuple tuple) { + if (tuple instanceof UnSafeTuple) { + UnSafeTuple unSafeTuple = TUtil.checkTypeAndGet(tuple, UnSafeTuple.class); + int length = unSafeTuple.getLength(); + ensureSize(length); + PlatformDependent.copyMemory(unSafeTuple.address(), address() + position(), length); + forward(length); + } else { + OffHeapRowBlockUtils.convert(tuple, this); + } + } +} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableLimitSpec.java similarity index 99% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableLimitSpec.java index 14e67b25eb..614b3fb01f 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableLimitSpec.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/RowBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/RowBlock.java new file mode 100644 index 0000000000..68902fbfea --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/RowBlock.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.tuple.memory; + +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.tuple.RowBlockReader; + +import java.io.IOException; +import java.nio.channels.ScatteringByteChannel; + +public interface RowBlock { + + void clear(); + + int capacity(); + + void setRows(int rowNum); + + int rows(); + + TajoDataTypes.DataType[] getDataTypes(); + + RowBlockReader getReader(); + + RowWriter getWriter(); + + MemoryBlock getMemory(); + + void release(); + + boolean copyFromChannel(ScatteringByteChannel channel) throws IOException; +} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/RowWriter.java similarity index 63% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/RowWriter.java index a2b256166d..0393714d3c 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/RowWriter.java @@ -16,11 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.IntervalDatum; import org.apache.tajo.datum.ProtobufDatum; +import org.apache.tajo.storage.Tuple; /** * The call sequence should be as follows: @@ -33,41 +34,47 @@ */ public interface RowWriter { - public TajoDataTypes.DataType [] dataTypes(); + TajoDataTypes.DataType [] dataTypes(); - public boolean startRow(); + boolean startRow(); - public void endRow(); + void endRow(); - public void skipField(); + void skipField(); - public void putBool(boolean val); + void clear(); - public void putInt2(short val); + void putByte(byte val); - public void putInt4(int val); + void putBool(boolean val); - public void putInt8(long val); + void putInt2(short val); - public void putFloat4(float val); + void putInt4(int val); - public void putFloat8(double val); + void putInt8(long val); - public void putText(String val); + void putFloat4(float val); - public void putText(byte[] val); + void putFloat8(double val); - public void putBlob(byte[] val); + void putText(String val); - public void putTimestamp(long val); + void putText(byte[] val); - public void putTime(long val); + void putBlob(byte[] val); - public void putDate(int val); + void putTimestamp(long val); - public void putInterval(IntervalDatum val); + void putTime(long val); - public void putInet4(int val); + void putDate(int val); - public void putProtoDatum(ProtobufDatum datum); + void putInterval(IntervalDatum val); + + void putInet4(int val); + + void putProtoDatum(ProtobufDatum datum); + + void addTuple(Tuple tuple); } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java similarity index 58% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java index 3756064eac..497613b038 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java @@ -16,11 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import com.google.common.base.Preconditions; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; +import io.netty.util.internal.PlatformDependent; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; import org.apache.tajo.exception.TajoRuntimeException; @@ -32,10 +33,8 @@ import org.apache.tajo.util.UnsafeUtil; import org.apache.tajo.util.datetime.TimeMeta; import sun.misc.Unsafe; -import sun.nio.ch.DirectBuffer; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.Charset; import static org.apache.tajo.common.TajoDataTypes.DataType; @@ -43,20 +42,25 @@ public abstract class UnSafeTuple implements Tuple { private static final Unsafe UNSAFE = UnsafeUtil.unsafe; - private DirectBuffer bb; + private long address; private int relativePos; private int length; private DataType [] types; - protected void set(ByteBuffer bb, int relativePos, int length, DataType [] types) { - this.bb = (DirectBuffer) bb; + protected void set(MemoryBlock memoryBlock, int relativePos, int length, DataType [] types) { + Preconditions.checkArgument(memoryBlock.hasAddress()); + + this.address = memoryBlock.address(); this.relativePos = relativePos; this.length = length; this.types = types; } - void set(ByteBuffer bb, DataType [] types) { - set(bb, 0, bb.limit(), types); + void set(UnSafeTuple tuple) { + this.address = tuple.address; + this.relativePos = tuple.relativePos; + this.length = tuple.length; + this.types = tuple.types; } @Override @@ -71,35 +75,42 @@ public TajoDataTypes.Type type(int fieldId) { @Override public int size(int fieldId) { - return UNSAFE.getInt(getFieldAddr(fieldId)); + return PlatformDependent.getInt(getFieldAddr(fieldId)); } - public ByteBuffer nioBuffer() { - return ((ByteBuffer)((ByteBuffer)bb).duplicate().position(relativePos).limit(relativePos + length)).slice(); - } + public void writeTo(ByteBuffer bb) { + if (bb.remaining() < length) { + throw new IndexOutOfBoundsException("remaining length: " + bb.remaining() + + ", tuple length: " + getLength()); + } - public HeapTuple toHeapTuple() { - byte [] bytes = new byte[length]; - UNSAFE.copyMemory(null, bb.address() + relativePos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, length); - return new HeapTuple(bytes, types); + if (length > 0) { + if (bb.isDirect()) { + PlatformDependent.copyMemory(address(), PlatformDependent.directBufferAddress(bb) + bb.position(), length); + bb.position(bb.position() + getLength()); + } else { + PlatformDependent.copyMemory(address(), bb.array(), bb.arrayOffset() + bb.position(), length); + bb.position(bb.position() + getLength()); + } + } } - public void copyFrom(UnSafeTuple tuple) { - Preconditions.checkNotNull(tuple); + public long address() { + return address + relativePos; + } - ((ByteBuffer) bb).clear(); - if (length < tuple.length) { - UnsafeUtil.free((ByteBuffer) bb); - bb = (DirectBuffer) ByteBuffer.allocateDirect(tuple.length).order(ByteOrder.nativeOrder()); - this.relativePos = 0; - this.length = tuple.length; - } + public int getLength() { + return length; + } - ((ByteBuffer) bb).put(tuple.nioBuffer()); + public HeapTuple toHeapTuple() { + byte [] bytes = new byte[length]; + PlatformDependent.copyMemory(address(), bytes, 0, length); + return new HeapTuple(bytes, types); } private int getFieldOffset(int fieldId) { - return UNSAFE.getInt(bb.address() + (long)(relativePos + SizeOf.SIZE_OF_INT + (fieldId * SizeOf.SIZE_OF_INT))); + return PlatformDependent.getInt(address()+ (long)(SizeOf.SIZE_OF_INT + (fieldId * SizeOf.SIZE_OF_INT))); } public long getFieldAddr(int fieldId) { @@ -107,22 +118,22 @@ public long getFieldAddr(int fieldId) { if (fieldOffset == -1) { throw new RuntimeException("Invalid Field Access: " + fieldId); } - return bb.address() + relativePos + fieldOffset; + return address() + fieldOffset; } @Override public boolean contains(int fieldid) { - return getFieldOffset(fieldid) > OffHeapRowBlock.NULL_FIELD_OFFSET; + return getFieldOffset(fieldid) > MemoryRowBlock.NULL_FIELD_OFFSET; } @Override public boolean isBlank(int fieldid) { - return getFieldOffset(fieldid) == OffHeapRowBlock.NULL_FIELD_OFFSET; + return getFieldOffset(fieldid) == MemoryRowBlock.NULL_FIELD_OFFSET; } @Override public boolean isBlankOrNull(int fieldid) { - return getFieldOffset(fieldid) == OffHeapRowBlock.NULL_FIELD_OFFSET; + return getFieldOffset(fieldid) == MemoryRowBlock.NULL_FIELD_OFFSET; } @Override @@ -152,35 +163,43 @@ public Datum asDatum(int fieldId) { } switch (types[fieldId].getType()) { - case BOOLEAN: - return DatumFactory.createBool(getBool(fieldId)); - case INT1: - case INT2: - return DatumFactory.createInt2(getInt2(fieldId)); - case INT4: - return DatumFactory.createInt4(getInt4(fieldId)); - case INT8: - return DatumFactory.createInt8(getInt4(fieldId)); - case FLOAT4: - return DatumFactory.createFloat4(getFloat4(fieldId)); - case FLOAT8: - return DatumFactory.createFloat8(getFloat8(fieldId)); - case TEXT: - return DatumFactory.createText(getText(fieldId)); - case TIMESTAMP: - return DatumFactory.createTimestamp(getInt8(fieldId)); - case DATE: - return DatumFactory.createDate(getInt4(fieldId)); - case TIME: - return DatumFactory.createTime(getInt8(fieldId)); - case INTERVAL: - return getInterval(fieldId); - case INET4: - return DatumFactory.createInet4(getInt4(fieldId)); - case PROTOBUF: - return getProtobufDatum(fieldId); - default: - throw new TajoRuntimeException(new UnsupportedException("data type '" + types[fieldId] + "'")); + case BOOLEAN: + return DatumFactory.createBool(getBool(fieldId)); + case BIT: + return DatumFactory.createBit(getByte(fieldId)); + case INT1: + case INT2: + return DatumFactory.createInt2(getInt2(fieldId)); + case INT4: + return DatumFactory.createInt4(getInt4(fieldId)); + case INT8: + return DatumFactory.createInt8(getInt8(fieldId)); + case FLOAT4: + return DatumFactory.createFloat4(getFloat4(fieldId)); + case FLOAT8: + return DatumFactory.createFloat8(getFloat8(fieldId)); + case CHAR: + return DatumFactory.createChar(getBytes(fieldId)); + case TEXT: + return DatumFactory.createText(getBytes(fieldId)); + case BLOB : + return DatumFactory.createBlob(getBytes(fieldId)); + case TIMESTAMP: + return DatumFactory.createTimestamp(getInt8(fieldId)); + case DATE: + return DatumFactory.createDate(getInt4(fieldId)); + case TIME: + return DatumFactory.createTime(getInt8(fieldId)); + case INTERVAL: + return getInterval(fieldId); + case INET4: + return DatumFactory.createInet4(getInt4(fieldId)); + case PROTOBUF: + return getProtobufDatum(fieldId); + case NULL_TYPE: + return NullDatum.get(); + default: + throw new TajoRuntimeException(new UnsupportedException("data type '" + types[fieldId] + "'")); } } @@ -199,12 +218,12 @@ public long getOffset() { @Override public boolean getBool(int fieldId) { - return UNSAFE.getByte(getFieldAddr(fieldId)) == 0x01; + return PlatformDependent.getByte(getFieldAddr(fieldId)) == 0x01; } @Override public byte getByte(int fieldId) { - return UNSAFE.getByte(getFieldAddr(fieldId)); + return PlatformDependent.getByte(getFieldAddr(fieldId)); } @Override @@ -215,61 +234,55 @@ public char getChar(int fieldId) { @Override public byte[] getBytes(int fieldId) { long pos = getFieldAddr(fieldId); - int len = UNSAFE.getInt(pos); + int len = PlatformDependent.getInt(pos); pos += SizeOf.SIZE_OF_INT; byte [] bytes = new byte[len]; - UNSAFE.copyMemory(null, pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len); + PlatformDependent.copyMemory(pos, bytes, 0, len); return bytes; } @Override public byte[] getTextBytes(int fieldId) { - long pos = getFieldAddr(fieldId); - int len = UNSAFE.getInt(pos); - pos += SizeOf.SIZE_OF_INT; - - byte[] bytes = new byte[len]; - UNSAFE.copyMemory(null, pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len); - return bytes; + return getBytes(fieldId); } @Override public short getInt2(int fieldId) { long addr = getFieldAddr(fieldId); - return UNSAFE.getShort(addr); + return PlatformDependent.getShort(addr); } @Override public int getInt4(int fieldId) { - return UNSAFE.getInt(getFieldAddr(fieldId)); + return PlatformDependent.getInt(getFieldAddr(fieldId)); } @Override public long getInt8(int fieldId) { - return UNSAFE.getLong(getFieldAddr(fieldId)); + return PlatformDependent.getLong(getFieldAddr(fieldId)); } @Override public float getFloat4(int fieldId) { - return UNSAFE.getFloat(getFieldAddr(fieldId)); + return Float.intBitsToFloat(PlatformDependent.getInt(getFieldAddr(fieldId))); } @Override public double getFloat8(int fieldId) { - return UNSAFE.getDouble(getFieldAddr(fieldId)); + return Double.longBitsToDouble(PlatformDependent.getLong(getFieldAddr(fieldId))); } @Override public String getText(int fieldId) { - return new String(getTextBytes(fieldId)); + return new String(getTextBytes(fieldId), TextDatum.DEFAULT_CHARSET); } public IntervalDatum getInterval(int fieldId) { long pos = getFieldAddr(fieldId); - int months = UNSAFE.getInt(pos); + int months = PlatformDependent.getInt(pos); pos += SizeOf.SIZE_OF_INT; - long millisecs = UNSAFE.getLong(pos); + long millisecs = PlatformDependent.getLong(pos); return new IntervalDatum(months, millisecs); } @@ -291,11 +304,11 @@ public Datum getProtobufDatum(int fieldId) { @Override public char[] getUnicodeChars(int fieldId) { long pos = getFieldAddr(fieldId); - int len = UNSAFE.getInt(pos); + int len = PlatformDependent.getInt(pos); pos += SizeOf.SIZE_OF_INT; byte [] bytes = new byte[len]; - UNSAFE.copyMemory(null, pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len); + PlatformDependent.copyMemory(pos, bytes, 0, len); return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8")); } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTupleBytesComparator.java similarity index 98% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTupleBytesComparator.java index 73e1e2fd3a..53a78a817a 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTupleBytesComparator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import com.google.common.primitives.Longs; import com.google.common.primitives.UnsignedLongs; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ZeroCopyTuple.java similarity index 78% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/ZeroCopyTuple.java index 51dbb295d5..3f2e832170 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ZeroCopyTuple.java @@ -16,16 +16,14 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; -import java.nio.ByteBuffer; - -import static org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.common.TajoDataTypes.DataType; public class ZeroCopyTuple extends UnSafeTuple { - public void set(ByteBuffer bb, int relativePos, int length, DataType [] types) { - super.set(bb, relativePos, length, types); + public void set(MemoryBlock memoryBlock, int relativePos, int length, DataType[] types) { + super.set(memoryBlock, relativePos, length, types); } @Override diff --git a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java b/tajo-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java similarity index 59% rename from tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java rename to tajo-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java index b332364423..8a3e59c633 100644 --- a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java @@ -18,59 +18,63 @@ package org.apache.tajo.tuple; -import org.apache.tajo.storage.RowStoreUtil; -import org.apache.tajo.tuple.offheap.*; +import org.apache.tajo.tuple.memory.*; import org.junit.Test; public class TestBaseTupleBuilder { @Test public void testBuild() { - BaseTupleBuilder builder = new BaseTupleBuilder(TestOffHeapRowBlock.schema); + BaseTupleBuilder builder = new BaseTupleBuilder(TestMemoryRowBlock.schema); - OffHeapRowBlock rowBlock = TestOffHeapRowBlock.createRowBlock(10248); - OffHeapRowBlockReader reader = rowBlock.getReader(); + MemoryRowBlock rowBlock = TestMemoryRowBlock.createRowBlock(10248); + RowBlockReader reader = rowBlock.getReader(); ZeroCopyTuple inputTuple = new ZeroCopyTuple(); - HeapTuple heapTuple = null; - ZeroCopyTuple zcTuple = null; + HeapTuple heapTuple; + ZeroCopyTuple zcTuple; int i = 0; while(reader.next(inputTuple)) { - RowStoreUtil.convert(inputTuple, builder); + OffHeapRowBlockUtils.convert(inputTuple, builder); heapTuple = builder.buildToHeapTuple(); - TestOffHeapRowBlock.validateTupleResult(i, heapTuple); + TestMemoryRowBlock.validateTupleResult(i, heapTuple); zcTuple = builder.buildToZeroCopyTuple(); - TestOffHeapRowBlock.validateTupleResult(i, zcTuple); + TestMemoryRowBlock.validateTupleResult(i, zcTuple); i++; } + builder.release(); + rowBlock.release(); } @Test public void testBuildWithNull() { - BaseTupleBuilder builder = new BaseTupleBuilder(TestOffHeapRowBlock.schema); + BaseTupleBuilder builder = new BaseTupleBuilder(TestMemoryRowBlock.schema); - OffHeapRowBlock rowBlock = TestOffHeapRowBlock.createRowBlockWithNull(10248); - OffHeapRowBlockReader reader = rowBlock.getReader(); + MemoryRowBlock rowBlock = TestMemoryRowBlock.createRowBlockWithNull(10248); + RowBlockReader reader = rowBlock.getReader(); ZeroCopyTuple inputTuple = new ZeroCopyTuple(); - HeapTuple heapTuple = null; - ZeroCopyTuple zcTuple = null; + HeapTuple heapTuple; + ZeroCopyTuple zcTuple; int i = 0; while(reader.next(inputTuple)) { - RowStoreUtil.convert(inputTuple, builder); + OffHeapRowBlockUtils.convert(inputTuple, builder); heapTuple = builder.buildToHeapTuple(); - TestOffHeapRowBlock.validateNullity(i, heapTuple); + TestMemoryRowBlock.validateNullity(i, heapTuple); zcTuple = builder.buildToZeroCopyTuple(); - TestOffHeapRowBlock.validateNullity(i, zcTuple); + TestMemoryRowBlock.validateNullity(i, zcTuple); i++; } + + builder.release(); + rowBlock.release(); } } \ No newline at end of file diff --git a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestHeapTuple.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java similarity index 72% rename from tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestHeapTuple.java rename to tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java index 96f465a59e..ac93d07da8 100644 --- a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestHeapTuple.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java @@ -16,27 +16,24 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; -import org.apache.tajo.catalog.SchemaUtil; import org.junit.Test; public class TestHeapTuple { @Test public void testHeapTuple() { - OffHeapRowBlock rowBlock = TestOffHeapRowBlock.createRowBlock(1024); + MemoryRowBlock rowBlock = TestMemoryRowBlock.createRowBlock(1024); OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); int i = 0; while (reader.next(zcTuple)) { - byte [] bytes = new byte[zcTuple.nioBuffer().limit()]; - zcTuple.nioBuffer().get(bytes); - HeapTuple heapTuple = new HeapTuple(bytes, SchemaUtil.toDataTypes(TestOffHeapRowBlock.schema)); - TestOffHeapRowBlock.validateTupleResult(i, heapTuple); + HeapTuple heapTuple = zcTuple.toHeapTuple(); + TestMemoryRowBlock.validateTupleResult(i, zcTuple); + TestMemoryRowBlock.validateTupleResult(i, heapTuple); i++; } diff --git a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java similarity index 82% rename from tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java rename to tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java index 278d733d35..5ace4464ba 100644 --- a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java @@ -16,58 +16,55 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.*; -import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.ProtobufDatum; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.NumberUtil; import org.apache.tajo.util.ProtoUtil; import org.junit.Test; -import java.nio.ByteBuffer; import java.util.Collections; +import java.util.Comparator; import java.util.List; import static org.apache.tajo.common.TajoDataTypes.Type; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class TestOffHeapRowBlock { - private static final Log LOG = LogFactory.getLog(TestOffHeapRowBlock.class); +public class TestMemoryRowBlock { + private static final Log LOG = LogFactory.getLog(TestMemoryRowBlock.class); public static String UNICODE_FIELD_PREFIX = "abc_가나다_"; - public static Schema schema; + public static DataType[] schema; static { - schema = new Schema(); - schema.addColumn("col0", Type.BOOLEAN); - schema.addColumn("col1", Type.INT2); - schema.addColumn("col2", Type.INT4); - schema.addColumn("col3", Type.INT8); - schema.addColumn("col4", Type.FLOAT4); - schema.addColumn("col5", Type.FLOAT8); - schema.addColumn("col6", Type.TEXT); - schema.addColumn("col7", Type.TIMESTAMP); - schema.addColumn("col8", Type.DATE); - schema.addColumn("col9", Type.TIME); - schema.addColumn("col10", Type.INTERVAL); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", - CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, PrimitiveProtos.StringProto.class.getName())); + schema = new DataType[] { + DataType.newBuilder().setType(Type.BOOLEAN).build(), + DataType.newBuilder().setType(Type.INT2).build(), + DataType.newBuilder().setType(Type.INT4).build(), + DataType.newBuilder().setType(Type.INT8).build(), + DataType.newBuilder().setType(Type.FLOAT4).build(), + DataType.newBuilder().setType(Type.FLOAT8).build(), + DataType.newBuilder().setType(Type.TEXT).build(), + DataType.newBuilder().setType(Type.TIMESTAMP).build(), + DataType.newBuilder().setType(Type.DATE).build(), + DataType.newBuilder().setType(Type.TIME).build(), + DataType.newBuilder().setType(Type.INTERVAL).build(), + DataType.newBuilder().setType(Type.INET4).build(), + DataType.newBuilder().setType(Type.PROTOBUF).build() + }; } - private void explainRowBlockAllocation(OffHeapRowBlock rowBlock, long startTime, long endTime) { - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " + private void explainRowBlockAllocation(MemoryRowBlock rowBlock, long startTime, long endTime) { + LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (endTime - startTime) + " msec"); } @@ -76,7 +73,7 @@ public void testPutAndReadValidation() { int rowNum = 1000; long allocStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, 1024); + MemoryRowBlock rowBlock = new MemoryRowBlock(schema, 1024); long allocEnd = System.currentTimeMillis(); explainRowBlockAllocation(rowBlock, allocStart, allocEnd); @@ -91,10 +88,10 @@ public void testPutAndReadValidation() { int j = 0; while(reader.next(tuple)) { validateTupleResult(j, tuple); - j++; } } + long writeEnd = System.currentTimeMillis(); LOG.info("writing and validating take " + (writeEnd - writeStart) + " msec"); @@ -106,6 +103,7 @@ public void testPutAndReadValidation() { validateTupleResult(j, tuple); j++; } + assertEquals(rowNum, j); long readEnd = System.currentTimeMillis(); LOG.info("reading takes " + (readEnd - readStart) + " msec"); @@ -117,7 +115,7 @@ public void testNullityValidation() { int rowNum = 1000; long allocStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, 1024); + MemoryRowBlock rowBlock = new MemoryRowBlock(schema, 1024); long allocEnd = System.currentTimeMillis(); explainRowBlockAllocation(rowBlock, allocStart, allocEnd); @@ -125,6 +123,7 @@ public void testNullityValidation() { ZeroCopyTuple tuple = new ZeroCopyTuple(); long writeStart = System.currentTimeMillis(); for (int i = 0; i < rowNum; i++) { + fillRowBlockWithNull(i, rowBlock.getWriter()); reader.reset(); @@ -136,7 +135,7 @@ public void testNullityValidation() { } } long writeEnd = System.currentTimeMillis(); - LOG.info("writing and nullity validating take " + (writeEnd - writeStart) +" msec"); + LOG.info("writing and nullity validating take " + (writeEnd - writeStart) + " msec"); long readStart = System.currentTimeMillis(); tuple = new ZeroCopyTuple(); @@ -147,6 +146,7 @@ public void testNullityValidation() { j++; } + assertEquals(rowNum, j); long readEnd = System.currentTimeMillis(); LOG.info("reading takes " + (readEnd - readStart) + " msec"); @@ -158,7 +158,7 @@ public void testEmptyRow() { int rowNum = 1000; long allocStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 10); + MemoryRowBlock rowBlock = new MemoryRowBlock(schema, StorageUnit.MB * 10); long allocEnd = System.currentTimeMillis(); explainRowBlockAllocation(rowBlock, allocStart, allocEnd); @@ -180,6 +180,8 @@ public void testEmptyRow() { while(reader.next(tuple)) { j++; } + + assertEquals(rowNum, j); long readEnd = System.currentTimeMillis(); LOG.info("reading takes " + (readEnd - readStart) + " msec"); rowBlock.release(); @@ -192,7 +194,7 @@ public void testEmptyRow() { public void testSortBenchmark() { int rowNum = 1000; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); List unSafeTuples = Lists.newArrayList(); @@ -207,11 +209,13 @@ public void testSortBenchmark() { long readEnd = System.currentTimeMillis(); LOG.info("reading takes " + (readEnd - readStart) + " msec"); - SortSpec sortSpec = new SortSpec(new Column("col2", Type.INT4)); - BaseTupleComparator comparator = new BaseTupleComparator(schema, new SortSpec[] {sortSpec}); - long sortStart = System.currentTimeMillis(); - Collections.sort(unSafeTuples, comparator); + Collections.sort(unSafeTuples, new Comparator() { + @Override + public int compare(ZeroCopyTuple t1, ZeroCopyTuple t2) { + return NumberUtil.compare(t1.getInt4(2), t2.getInt4(2)); + } + }); long sortEnd = System.currentTimeMillis(); LOG.info("sorting took " + (sortEnd - sortStart) + " msec"); rowBlock.release(); @@ -225,7 +229,7 @@ public void testVTuplePutAndGetBenchmark() { long writeStart = System.currentTimeMillis(); VTuple tuple; for (int i = 0; i < rowNum; i++) { - tuple = new VTuple(schema.size()); + tuple = new VTuple(schema.length); fillVTuple(i, tuple); rowBlock.add(tuple); } @@ -238,12 +242,14 @@ public void testVTuplePutAndGetBenchmark() { validateTupleResult(j, t); j++; } + + assertEquals(rowNum, j); long readEnd = System.currentTimeMillis(); LOG.info("reading takes " + (readEnd - readStart) + " msec"); int count = 0; for (int l = 0; l < rowBlock.size(); l++) { - for(int m = 0; m < schema.size(); m++ ) { + for(int m = 0; m < schema.length; m++ ) { if (rowBlock.get(l).contains(m) && rowBlock.get(l).get(m).type() == Type.INT4) { count ++; } @@ -257,14 +263,13 @@ public void testVTuplePutAndGetBenchmark() { public void testVTuplePutAndGetBenchmarkViaDirectRowEncoder() { int rowNum = 1000; - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 100); + MemoryRowBlock rowBlock = new MemoryRowBlock(schema, StorageUnit.MB * 100); long writeStart = System.currentTimeMillis(); - VTuple tuple = new VTuple(schema.size()); + VTuple tuple = new VTuple(schema.length); for (int i = 0; i < rowNum; i++) { fillVTuple(i, tuple); - - RowStoreUtil.convert(tuple, rowBlock.getWriter()); + rowBlock.getWriter().addTuple(tuple); } long writeEnd = System.currentTimeMillis(); LOG.info("Writing takes " + (writeEnd - writeStart) + " msec"); @@ -277,10 +282,9 @@ public void testVTuplePutAndGetBenchmarkViaDirectRowEncoder() { public void testSerDerOfRowBlock() { int rowNum = 1000; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); - ByteBuffer bb = rowBlock.nioBuffer(); - OffHeapRowBlock restoredRowBlock = new OffHeapRowBlock(schema, bb); + MemoryRowBlock restoredRowBlock = new MemoryRowBlock(rowBlock); validateResults(restoredRowBlock); rowBlock.release(); } @@ -289,36 +293,45 @@ public void testSerDerOfRowBlock() { public void testSerDerOfZeroCopyTuple() { int rowNum = 1000; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); - ByteBuffer bb = rowBlock.nioBuffer(); - OffHeapRowBlock restoredRowBlock = new OffHeapRowBlock(schema, bb); + MemoryRowBlock restoredRowBlock = new MemoryRowBlock(rowBlock); OffHeapRowBlockReader reader = new OffHeapRowBlockReader(restoredRowBlock); long readStart = System.currentTimeMillis(); ZeroCopyTuple tuple = new ZeroCopyTuple(); - ZeroCopyTuple copyTuple = new ZeroCopyTuple(); + int j = 0; + List copyTuples = Lists.newArrayList(); reader.reset(); - while(reader.next(tuple)) { - ByteBuffer copy = tuple.nioBuffer(); - copyTuple.set(copy, SchemaUtil.toDataTypes(schema)); - validateTupleResult(j, copyTuple); + while (reader.next(tuple)) { + validateTupleResult(j, tuple); + + ZeroCopyTuple copyTuple = new ZeroCopyTuple(); + copyTuple.set(tuple); + copyTuples.add(copyTuple); j++; } + + assertEquals(rowNum, j); + + for (int i = 0; i < j; i++) { + validateTupleResult(i, copyTuples.get(i)); + } + long readEnd = System.currentTimeMillis(); LOG.info("reading takes " + (readEnd - readStart) + " msec"); rowBlock.release(); } - public static OffHeapRowBlock createRowBlock(int rowNum) { + public static MemoryRowBlock createRowBlock(int rowNum) { long allocateStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 8); + MemoryRowBlock rowBlock = new MemoryRowBlock(schema, StorageUnit.MB * 8); long allocatedEnd = System.currentTimeMillis(); - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " + LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (allocatedEnd - allocateStart) + " msec"); long writeStart = System.currentTimeMillis(); @@ -331,11 +344,11 @@ public static OffHeapRowBlock createRowBlock(int rowNum) { return rowBlock; } - public static OffHeapRowBlock createRowBlockWithNull(int rowNum) { + public static MemoryRowBlock createRowBlockWithNull(int rowNum) { long allocateStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 8); + MemoryRowBlock rowBlock = new MemoryRowBlock(schema, StorageUnit.MB * 8); long allocatedEnd = System.currentTimeMillis(); - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " + LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (allocatedEnd - allocateStart) + " msec"); long writeStart = System.currentTimeMillis(); @@ -465,7 +478,7 @@ public static void fillVTuple(int i, VTuple tuple) { tuple.put(12, new ProtobufDatum(ProtoUtil.convertString(i + ""))); // 12; } - public static void validateResults(OffHeapRowBlock rowBlock) { + public static void validateResults(MemoryRowBlock rowBlock) { long readStart = System.currentTimeMillis(); ZeroCopyTuple tuple = new ZeroCopyTuple(); int j = 0; diff --git a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestResizableSpec.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestResizableSpec.java similarity index 98% rename from tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestResizableSpec.java rename to tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestResizableSpec.java index 1eb9c17d07..483dad503f 100644 --- a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/tuple/offheap/TestResizableSpec.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestResizableSpec.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.tuple.offheap; +package org.apache.tajo.tuple.memory; import org.apache.tajo.unit.StorageUnit; import org.junit.Test; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java index 7708d529db..8ca55cc112 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java @@ -23,11 +23,9 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.IntervalDatum; -import org.apache.tajo.datum.ProtobufDatum; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.exception.ValueTooLongForTypeCharactersException; -import org.apache.tajo.tuple.offheap.RowWriter; import org.apache.tajo.util.BitArray; import java.nio.ByteBuffer; @@ -335,56 +333,4 @@ public Schema getSchema() { return schema; } } - - public static void convert(Tuple tuple, RowWriter writer) { - writer.startRow(); - - for (int i = 0; i < writer.dataTypes().length; i++) { - if (tuple.isBlankOrNull(i)) { - writer.skipField(); - continue; - } - switch (writer.dataTypes()[i].getType()) { - case BOOLEAN: - writer.putBool(tuple.getBool(i)); - break; - case INT1: - case INT2: - writer.putInt2(tuple.getInt2(i)); - break; - case INT4: - case DATE: - case INET4: - writer.putInt4(tuple.getInt4(i)); - break; - case INT8: - case TIMESTAMP: - case TIME: - writer.putInt8(tuple.getInt8(i)); - break; - case FLOAT4: - writer.putFloat4(tuple.getFloat4(i)); - break; - case FLOAT8: - writer.putFloat8(tuple.getFloat8(i)); - break; - case TEXT: - writer.putText(tuple.getBytes(i)); - break; - case INTERVAL: - writer.putInterval((IntervalDatum) tuple.getInterval(i)); - break; - case PROTOBUF: - writer.putProtoDatum((ProtobufDatum) tuple.getProtobufDatum(i)); - break; - case NULL_TYPE: - writer.skipField(); - break; - default: - throw new TajoRuntimeException( - new UnsupportedException("unknown data type '" + writer.dataTypes()[i].getType().name() + "'")); - } - } - writer.endRow(); - } } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java deleted file mode 100644 index 2f8e3494ca..0000000000 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple.offheap; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.util.Deallocatable; -import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.UnsafeUtil; -import sun.misc.Unsafe; -import sun.nio.ch.DirectBuffer; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public class OffHeapMemory implements Deallocatable { - private static final Log LOG = LogFactory.getLog(OffHeapMemory.class); - - protected static final Unsafe UNSAFE = UnsafeUtil.unsafe; - - protected ByteBuffer buffer; - protected int memorySize; - protected ResizableLimitSpec limitSpec; - protected long address; - - @VisibleForTesting - protected OffHeapMemory(ByteBuffer buffer, ResizableLimitSpec limitSpec) { - this.buffer = buffer; - this.address = ((DirectBuffer) buffer).address(); - this.memorySize = buffer.limit(); - this.limitSpec = limitSpec; - } - - public OffHeapMemory(ResizableLimitSpec limitSpec) { - this(ByteBuffer.allocateDirect((int) limitSpec.initialSize()).order(ByteOrder.nativeOrder()), limitSpec); - } - - public long address() { - return address; - } - - public long size() { - return memorySize; - } - - public void resize(int newSize) { - Preconditions.checkArgument(newSize > 0, "Size must be greater than 0 bytes"); - - if (newSize > limitSpec.limit()) { - throw new RuntimeException("Resize cannot exceed the size limit"); - } - - if (newSize < memorySize) { - LOG.warn("The size reduction is ignored."); - } - - int newBlockSize = UnsafeUtil.alignedSize(newSize); - ByteBuffer newByteBuf = ByteBuffer.allocateDirect(newBlockSize); - long newAddress = ((DirectBuffer)newByteBuf).address(); - - UNSAFE.copyMemory(this.address, newAddress, memorySize); - - UnsafeUtil.free(buffer); - this.memorySize = newSize; - this.buffer = newByteBuf; - this.address = newAddress; - } - - public java.nio.Buffer nioBuffer() { - return (ByteBuffer) buffer.position(0).limit(memorySize); - } - - @Override - public void release() { - UnsafeUtil.free(this.buffer); - this.buffer = null; - this.address = 0; - this.memorySize = 0; - } - - public String toString() { - return "memory=" + FileUtil.humanReadableByteCount(memorySize, false) + "," + limitSpec; - } -} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java deleted file mode 100644 index 90d47918d6..0000000000 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java +++ /dev/null @@ -1,213 +0,0 @@ -/*** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple.offheap; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaUtil; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.storage.SeekableInputChannel; -import org.apache.tajo.util.Deallocatable; -import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.SizeOf; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; - -import static org.apache.tajo.common.TajoDataTypes.DataType; - -public class OffHeapRowBlock extends OffHeapMemory implements Deallocatable { - private static final Log LOG = LogFactory.getLog(OffHeapRowBlock.class); - - public static final int NULL_FIELD_OFFSET = -1; - - DataType [] dataTypes; - - // Basic States - private int maxRowNum = Integer.MAX_VALUE; // optional - private int rowNum; - protected int position = 0; - - private OffHeapRowBlockWriter builder; - - private OffHeapRowBlock(ByteBuffer buffer, Schema schema, ResizableLimitSpec limitSpec) { - super(buffer, limitSpec); - initialize(schema); - } - - public OffHeapRowBlock(Schema schema, ResizableLimitSpec limitSpec) { - super(limitSpec); - initialize(schema); - } - - private void initialize(Schema schema) { - dataTypes = SchemaUtil.toDataTypes(schema); - - this.builder = new OffHeapRowBlockWriter(this); - } - - @VisibleForTesting - public OffHeapRowBlock(Schema schema, int bytes) { - this(schema, new ResizableLimitSpec(bytes)); - } - - @VisibleForTesting - public OffHeapRowBlock(Schema schema, ByteBuffer buffer) { - this(buffer, schema, ResizableLimitSpec.DEFAULT_LIMIT); - } - - public void position(int pos) { - this.position = pos; - } - - public void clear() { - this.position = 0; - this.rowNum = 0; - - builder.clear(); - } - - @Override - public ByteBuffer nioBuffer() { - return (ByteBuffer) buffer.position(0).limit(position); - } - - public int position() { - return position; - } - - public long usedMem() { - return position; - } - - /** - * Ensure that this buffer has enough remaining space to add the size. - * Creates and copies to a new buffer if necessary - * - * @param size Size to add - */ - public void ensureSize(int size) { - if (remain() - size < 0) { - if (!limitSpec.canIncrease(memorySize)) { - throw new RuntimeException("Cannot increase RowBlock anymore."); - } - - int newBlockSize = limitSpec.increasedSize(memorySize); - resize(newBlockSize); - LOG.info("Increase DirectRowBlock to " + FileUtil.humanReadableByteCount(newBlockSize, false)); - } - } - - public long remain() { - return memorySize - position - builder.offset(); - } - - public int maxRowNum() { - return maxRowNum; - } - public int rows() { - return rowNum; - } - - public void setRows(int rowNum) { - this.rowNum = rowNum; - } - - - public boolean copyFromChannel(SeekableInputChannel channel, TableStats stats) throws IOException { - if (channel.position() < channel.size()) { - clear(); - - buffer.clear(); - channel.read(buffer); - memorySize = buffer.position(); - - while (position < memorySize) { - long recordPtr = address + position; - - if (remain() < SizeOf.SIZE_OF_INT) { - channel.seek(channel.position() - remain()); - memorySize = (int) (memorySize - remain()); - return true; - } - - int recordSize = UNSAFE.getInt(recordPtr); - - if (remain() < recordSize) { - channel.seek(channel.position() - remain()); - memorySize = (int) (memorySize - remain()); - return true; - } - - position += recordSize; - rowNum++; - } - - return true; - } else { - return false; - } - } - - public boolean copyFromChannel(FileChannel channel, TableStats stats) throws IOException { - if (channel.position() < channel.size()) { - clear(); - - buffer.clear(); - channel.read(buffer); - memorySize = buffer.position(); - - while (position < memorySize) { - long recordPtr = address + position; - - if (remain() < SizeOf.SIZE_OF_INT) { - channel.position(channel.position() - remain()); - memorySize = (int) (memorySize - remain()); - return true; - } - - int recordSize = UNSAFE.getInt(recordPtr); - - if (remain() < recordSize) { - channel.position(channel.position() - remain()); - memorySize = (int) (memorySize - remain()); - return true; - } - - position += recordSize; - rowNum++; - } - - return true; - } else { - return false; - } - } - - public RowWriter getWriter() { - return builder; - } - - public OffHeapRowBlockReader getReader() { - return new OffHeapRowBlockReader(this); - } -} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java deleted file mode 100644 index dbc31883bd..0000000000 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple.offheap; - -import com.google.common.collect.Lists; -import org.apache.tajo.storage.Tuple; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -public class OffHeapRowBlockUtils { - - public static List sort(OffHeapRowBlock rowBlock, Comparator comparator) { - List tupleList = Lists.newArrayList(); - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - while(reader.next(zcTuple)) { - tupleList.add(zcTuple); - zcTuple = new ZeroCopyTuple(); - } - Collections.sort(tupleList, comparator); - return tupleList; - } - - public static Tuple[] sortToArray(OffHeapRowBlock rowBlock, Comparator comparator) { - Tuple[] tuples = new Tuple[rowBlock.rows()]; - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - for (int i = 0; i < rowBlock.rows() && reader.next(zcTuple); i++) { - tuples[i] = zcTuple; - zcTuple = new ZeroCopyTuple(); - } - Arrays.sort(tuples, comparator); - return tuples; - } -} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java deleted file mode 100644 index 85c7e0bf13..0000000000 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java +++ /dev/null @@ -1,232 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple.offheap; - -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.IntervalDatum; -import org.apache.tajo.datum.ProtobufDatum; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.util.SizeOf; -import org.apache.tajo.util.UnsafeUtil; - -/** - * - * Row Record Structure - * - * | row length (4 bytes) | field 1 offset | field 2 offset | ... | field N offset| field 1 | field 2| ... | field N | - * 4 bytes 4 bytes 4 bytes - * - */ -public abstract class OffHeapRowWriter implements RowWriter { - /** record size + offset list */ - private final int headerSize; - /** field offsets */ - private final int [] fieldOffsets; - private final TajoDataTypes.DataType [] dataTypes; - - private int curFieldIdx; - private int curOffset; - - public OffHeapRowWriter(final TajoDataTypes.DataType [] dataTypes) { - this.dataTypes = dataTypes; - fieldOffsets = new int[dataTypes.length]; - headerSize = SizeOf.SIZE_OF_INT * (dataTypes.length + 1); - } - - public void clear() { - curOffset = 0; - curFieldIdx = 0; - } - - public long recordStartAddr() { - return address() + position(); - } - - public abstract long address(); - - public abstract void ensureSize(int size); - - public int offset() { - return curOffset; - } - - /** - * Current position - * - * @return The position - */ - public abstract int position(); - - /** - * Forward the address; - * - * @param length Length to be forwarded - */ - public abstract void forward(int length); - - @Override - public TajoDataTypes.DataType[] dataTypes() { - return dataTypes; - } - - public boolean startRow() { - curOffset = headerSize; - curFieldIdx = 0; - return true; - } - - public void endRow() { - long rowHeaderPos = address() + position(); - OffHeapMemory.UNSAFE.putInt(rowHeaderPos, curOffset); - rowHeaderPos += SizeOf.SIZE_OF_INT; - - for (int i = 0; i < curFieldIdx; i++) { - OffHeapMemory.UNSAFE.putInt(rowHeaderPos, fieldOffsets[i]); - rowHeaderPos += SizeOf.SIZE_OF_INT; - } - for (int i = curFieldIdx; i < dataTypes.length; i++) { - OffHeapMemory.UNSAFE.putInt(rowHeaderPos, OffHeapRowBlock.NULL_FIELD_OFFSET); - rowHeaderPos += SizeOf.SIZE_OF_INT; - } - - // rowOffset is equivalent to a byte length of this row. - forward(curOffset); - } - - public void skipField() { - fieldOffsets[curFieldIdx++] = OffHeapRowBlock.NULL_FIELD_OFFSET; - } - - private void forwardField() { - fieldOffsets[curFieldIdx++] = curOffset; - } - - public void putBool(boolean val) { - ensureSize(SizeOf.SIZE_OF_BOOL); - forwardField(); - - OffHeapMemory.UNSAFE.putByte(recordStartAddr() + curOffset, (byte) (val ? 0x01 : 0x00)); - - curOffset += SizeOf.SIZE_OF_BOOL; - } - - public void putInt2(short val) { - ensureSize(SizeOf.SIZE_OF_SHORT); - forwardField(); - - OffHeapMemory.UNSAFE.putShort(recordStartAddr() + curOffset, val); - curOffset += SizeOf.SIZE_OF_SHORT; - } - - public void putInt4(int val) { - ensureSize(SizeOf.SIZE_OF_INT); - forwardField(); - - OffHeapMemory.UNSAFE.putInt(recordStartAddr() + curOffset, val); - curOffset += SizeOf.SIZE_OF_INT; - } - - public void putInt8(long val) { - ensureSize(SizeOf.SIZE_OF_LONG); - forwardField(); - - OffHeapMemory.UNSAFE.putLong(recordStartAddr() + curOffset, val); - curOffset += SizeOf.SIZE_OF_LONG; - } - - public void putFloat4(float val) { - ensureSize(SizeOf.SIZE_OF_FLOAT); - forwardField(); - - OffHeapMemory.UNSAFE.putFloat(recordStartAddr() + curOffset, val); - curOffset += SizeOf.SIZE_OF_FLOAT; - } - - public void putFloat8(double val) { - ensureSize(SizeOf.SIZE_OF_DOUBLE); - forwardField(); - - OffHeapMemory.UNSAFE.putDouble(recordStartAddr() + curOffset, val); - curOffset += SizeOf.SIZE_OF_DOUBLE; - } - - public void putText(String val) { - byte[] bytes = val.getBytes(TextDatum.DEFAULT_CHARSET); - putText(bytes); - } - - public void putText(byte[] val) { - int bytesLen = val.length; - - ensureSize(SizeOf.SIZE_OF_INT + bytesLen); - forwardField(); - - OffHeapMemory.UNSAFE.putInt(recordStartAddr() + curOffset, bytesLen); - curOffset += SizeOf.SIZE_OF_INT; - - OffHeapMemory.UNSAFE.copyMemory(val, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, null, - recordStartAddr() + curOffset, bytesLen); - curOffset += bytesLen; - } - - public void putBlob(byte[] val) { - int bytesLen = val.length; - - ensureSize(SizeOf.SIZE_OF_INT + bytesLen); - forwardField(); - - OffHeapMemory.UNSAFE.putInt(recordStartAddr() + curOffset, bytesLen); - curOffset += SizeOf.SIZE_OF_INT; - - OffHeapMemory.UNSAFE.copyMemory(val, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, null, - recordStartAddr() + curOffset, bytesLen); - curOffset += bytesLen; - } - - public void putTimestamp(long val) { - putInt8(val); - } - - public void putDate(int val) { - putInt4(val); - } - - public void putTime(long val) { - putInt8(val); - } - - public void putInterval(IntervalDatum val) { - ensureSize(SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_LONG); - forwardField(); - - long offset = recordStartAddr() + curOffset; - OffHeapMemory.UNSAFE.putInt(offset, val.getMonths()); - offset += SizeOf.SIZE_OF_INT; - OffHeapMemory.UNSAFE.putLong(offset, val.getMilliSeconds()); - curOffset += SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_LONG; - } - - public void putInet4(int val) { - putInt4(val); - } - - public void putProtoDatum(ProtobufDatum val) { - putBlob(val.asByteArray()); - } -} diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index dfdff85e85..676c072d7a 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -39,7 +39,7 @@ tajo.storage.scanner-handler - text,json,raw,rcfile,row,parquet,orc,sequencefile,avro,hbase + text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase @@ -55,6 +55,10 @@ tajo.storage.fragment.raw.class org.apache.tajo.storage.fragment.FileFragment + + tajo.storage.fragment.draw.class + org.apache.tajo.storage.fragment.FileFragment + tajo.storage.fragment.rcfile.class org.apache.tajo.storage.fragment.FileFragment @@ -100,6 +104,11 @@ org.apache.tajo.storage.RawFile$RawFileScanner + + tajo.storage.scanner-handler.draw.class + org.apache.tajo.storage.rawfile.DirectRawFileScanner + + tajo.storage.scanner-handler.rcfile.class org.apache.tajo.storage.rcfile.RCFile$RCFileScanner @@ -134,7 +143,7 @@ tajo.storage.scanner-handler.hbase.class org.apache.tajo.storage.hbase.HBaseScanner - + tajo.storage.appender-handler @@ -156,6 +165,11 @@ org.apache.tajo.storage.RawFile$RawFileAppender + + tajo.storage.appender-handler.draw.class + org.apache.tajo.storage.rawfile.DirectRawFileWriter + + tajo.storage.appender-handler.rcfile.class org.apache.tajo.storage.rcfile.RCFile$RCFileAppender @@ -212,4 +226,4 @@ 131072 128KB write buffer - + \ No newline at end of file diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml index f637da0d94..8a9b9ea970 100644 --- a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml @@ -38,7 +38,7 @@ tajo.storage.scanner-handler - text,json,raw,rcfile,row,parquet,orc,sequencefile,avro,hbase + text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro,hbase @@ -54,6 +54,10 @@ tajo.storage.fragment.raw.class org.apache.tajo.storage.fragment.FileFragment + + tajo.storage.fragment.draw.class + org.apache.tajo.storage.fragment.FileFragment + tajo.storage.fragment.rcfile.class org.apache.tajo.storage.fragment.FileFragment @@ -99,6 +103,11 @@ org.apache.tajo.storage.RawFile$RawFileScanner + + tajo.storage.scanner-handler.draw.class + org.apache.tajo.storage.rawfile.DirectRawFileScanner + + tajo.storage.scanner-handler.rcfile.class org.apache.tajo.storage.rcfile.RCFile$RCFileScanner @@ -116,7 +125,7 @@ tajo.storage.scanner-handler.orc.class - org.apache.tajo.storage.orc.OrcScanner + org.apache.tajo.storage.orc.ORCScanner @@ -137,7 +146,7 @@ tajo.storage.appender-handler - text,raw,rcfile,row,parquet,sequencefile,avro,hbase + text,raw,draw,rcfile,row,parquet,sequencefile,avro,hbase @@ -155,6 +164,11 @@ org.apache.tajo.storage.RawFile$RawFileAppender + + tajo.storage.appender-handler.draw.class + org.apache.tajo.storage.rawfile.DirectRawFileWriter + + tajo.storage.appender-handler.rcfile.class org.apache.tajo.storage.rcfile.RCFile$RCFileAppender @@ -211,4 +225,4 @@ 131072 128KB write buffer - + \ No newline at end of file diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java index 8ae9a263c1..bc2df79d06 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java @@ -25,15 +25,16 @@ import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.io.IOUtils; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.tuple.offheap.OffHeapRowBlock; -import org.apache.tajo.tuple.offheap.OffHeapRowBlockReader; -import org.apache.tajo.tuple.offheap.ZeroCopyTuple; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.tuple.RowBlockReader; +import org.apache.tajo.tuple.memory.MemoryRowBlock; +import org.apache.tajo.tuple.memory.RowBlock; +import org.apache.tajo.tuple.memory.ZeroCopyTuple; import org.apache.tajo.unit.StorageUnit; import java.io.File; @@ -44,30 +45,26 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner private static final Log LOG = LogFactory.getLog(DirectRawFileScanner.class); private SeekableInputChannel channel; - private TajoDataTypes.DataType[] columnTypes; + //private DataType[] columnTypes; private boolean eof = false; private long fileSize; private long recordCount; private ZeroCopyTuple unSafeTuple = new ZeroCopyTuple(); - private OffHeapRowBlock tupleBuffer; - private OffHeapRowBlockReader reader; + private RowBlock tupleBuffer; + private RowBlockReader reader; - public DirectRawFileScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException { + public DirectRawFileScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { super(conf, schema, meta, fragment); } public void init() throws IOException { initChannel(); - columnTypes = new TajoDataTypes.DataType[schema.size()]; - for (int i = 0; i < schema.size(); i++) { - columnTypes[i] = schema.getColumn(i).getDataType(); - } + tupleBuffer = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), 64 * StorageUnit.KB); - tupleBuffer = new OffHeapRowBlock(schema, 64 * StorageUnit.KB); - reader = new OffHeapRowBlockReader(tupleBuffer); + reader = tupleBuffer.getReader(); fetchNeeded = !next(tupleBuffer); @@ -104,7 +101,7 @@ private void initChannel() throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("RawFileScanner open:" + fragment.getPath() + ", offset :" + - fragment.getStartKey() + ", file size :" + fileSize); + fragment.getStartKey() + ", file capacity :" + fileSize); } } @@ -119,8 +116,8 @@ public void seek(long offset) throws IOException { fetchNeeded = true; } - public boolean next(OffHeapRowBlock rowblock) throws IOException { - return rowblock.copyFromChannel(channel, tableStats); + public boolean next(RowBlock rowblock) throws IOException { + return rowblock.copyFromChannel(channel); } private boolean fetchNeeded = true; @@ -162,8 +159,10 @@ public void close() throws IOException { tableStats.setReadBytes(fileSize); tableStats.setNumRows(recordCount); } - tupleBuffer.release(); - tupleBuffer = null; + if(tupleBuffer != null) { + tupleBuffer.release(); + tupleBuffer = null; + } reader = null; IOUtils.cleanup(LOG, channel); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java index 912649f7e1..25b921b8f5 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java @@ -28,21 +28,21 @@ import org.apache.hadoop.io.IOUtils; import org.apache.tajo.TaskAttemptId; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.plan.serder.PlanProto.ShuffleType; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.*; -import org.apache.tajo.tuple.BaseTupleBuilder; -import org.apache.tajo.tuple.offheap.OffHeapRowBlock; -import org.apache.tajo.tuple.offheap.UnSafeTuple; +import org.apache.tajo.storage.FileAppender; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.TableStatistics; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.tuple.memory.MemoryRowBlock; import org.apache.tajo.unit.StorageUnit; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.ByteBuffer; import java.nio.channels.FileChannel; public class DirectRawFileWriter extends FileAppender { @@ -52,14 +52,12 @@ public class DirectRawFileWriter extends FileAppender { private FileChannel channel; private RandomAccessFile randomAccessFile; private FSDataOutputStream fos; - private TajoDataTypes.DataType[] columnTypes; private boolean isLocal; private long pos; private TableStatistics stats; private ShuffleType shuffleType; - - private BaseTupleBuilder builder; + private MemoryRowBlock memoryRowBlock; public DirectRawFileWriter(Configuration conf, TaskAttemptId taskAttemptId, final Schema schema, final TableMeta meta, final Path path) throws IOException { @@ -90,12 +88,6 @@ public void init() throws IOException { isLocal = false; } - pos = 0; - columnTypes = new TajoDataTypes.DataType[schema.size()]; - for (int i = 0; i < schema.size(); i++) { - columnTypes[i] = schema.getColumn(i).getDataType(); - } - if (enabledStats) { this.stats = new TableStatistics(this.schema); this.shuffleType = PlannerUtil.getShuffleType( @@ -103,8 +95,8 @@ public void init() throws IOException { PlannerUtil.getShuffleType(ShuffleType.NONE_SHUFFLE))); } - builder = new BaseTupleBuilder(schema); - + memoryRowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), 64 * StorageUnit.KB); + pos = 0; super.init(); } @@ -121,34 +113,17 @@ private long getFilePosition() throws IOException { } } - public void writeRowBlock(OffHeapRowBlock rowBlock) throws IOException { - write(rowBlock.nioBuffer()); - if (enabledStats) { - stats.incrementRows(rowBlock.rows()); + public void writeRowBlock(MemoryRowBlock rowBlock) throws IOException { + if(isLocal) { + pos += rowBlock.getMemory().writeTo(channel); + } else { + pos += rowBlock.getMemory().writeTo(fos); } - pos = getFilePosition(); - } - - private ByteBuffer buffer; - private void ensureSize(int size) throws IOException { - if (buffer.remaining() < size) { - - buffer.limit(buffer.position()); - buffer.flip(); - write(buffer); - - buffer.clear(); - } - } + rowBlock.getMemory().clear(); - private void write(ByteBuffer buffer) throws IOException { - if(isLocal) { - channel.write(buffer); - } else { - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - fos.write(bytes); + if (enabledStats) { + stats.incrementRows(rowBlock.rows()); } } @@ -161,43 +136,23 @@ public void addTuple(Tuple t) throws IOException { } } - if (buffer == null) { - buffer = ByteBuffer.allocateDirect(64 * StorageUnit.KB); - } - - UnSafeTuple unSafeTuple; - - if (!(t instanceof UnSafeTuple)) { - RowStoreUtil.convert(t, builder); - unSafeTuple = builder.buildToZeroCopyTuple(); - } else { - unSafeTuple = (UnSafeTuple) t; - } - - ByteBuffer bb = unSafeTuple.nioBuffer(); - ensureSize(bb.limit()); - buffer.put(bb); - - pos = getFilePosition() + (buffer.limit() - buffer.remaining()); - - if (enabledStats) { - stats.incrementRow(); + memoryRowBlock.getWriter().addTuple(t); + if(memoryRowBlock.getMemory().readableBytes() > 64 * StorageUnit.KB) { + writeRowBlock(memoryRowBlock); } } @Override public void flush() throws IOException { - if (buffer != null) { - buffer.limit(buffer.position()); - buffer.flip(); - write(buffer); - buffer.clear(); + if(memoryRowBlock.getMemory().isReadable()) { + writeRowBlock(memoryRowBlock); } } @Override public void close() throws IOException { flush(); + if (enabledStats) { stats.setNumBytes(getOffset()); } @@ -206,6 +161,7 @@ public void close() throws IOException { } IOUtils.cleanup(LOG, channel, randomAccessFile, fos); + memoryRowBlock.release(); } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java index 3c667bf9f0..2456907baf 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java @@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicInteger; public class ByteBufLineReader implements Closeable { - public static int DEFAULT_BUFFER = 64 * 1024; + public static final int DEFAULT_BUFFER = 64 * 1024; private int bufferSize; private long readBytes; @@ -100,7 +100,7 @@ private void fillBuffer() throws IOException { tailBytes = this.buffer.writerIndex(); if (!this.buffer.isWritable()) { // a line bytes is large than the buffer - BufferPool.ensureWritable(buffer, bufferSize * 2); + this.buffer = BufferPool.ensureWritable(buffer, bufferSize * 2); this.bufferSize = buffer.capacity(); } this.startIndex = 0; diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index afe0f13265..bc9a6245d8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -25,6 +25,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.QueryId; import org.apache.tajo.TajoIdProtos; import org.apache.tajo.catalog.CatalogUtil; @@ -58,41 +59,41 @@ @RunWith(Parameterized.class) public class TestStorages { - private TajoConf conf; - private static String TEST_PATH = "target/test-data/TestStorages"; + private TajoConf conf; + private static String TEST_PATH = "target/test-data/TestStorages"; private static String TEST_PROJECTION_AVRO_SCHEMA = "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testProjection\",\n" + - " \"fields\": [\n" + - " { \"name\": \"id\", \"type\": \"int\" },\n" + - " { \"name\": \"age\", \"type\": \"long\" },\n" + - " { \"name\": \"score\", \"type\": \"float\" }\n" + - " ]\n" + - "}\n"; + " \"type\": \"record\",\n" + + " \"namespace\": \"org.apache.tajo\",\n" + + " \"name\": \"testProjection\",\n" + + " \"fields\": [\n" + + " { \"name\": \"id\", \"type\": \"int\" },\n" + + " { \"name\": \"age\", \"type\": \"long\" },\n" + + " { \"name\": \"score\", \"type\": \"float\" }\n" + + " ]\n" + + "}\n"; private static String TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA = "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testNullHandlingTypes\",\n" + - " \"fields\": [\n" + - " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" + - " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" + - " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" + - " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" + - " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" + - " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" + - " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" + - " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" + - " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" + - " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" + - " { \"name\": \"col11\", \"type\": \"null\" },\n" + - " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" + - " ]\n" + - "}\n"; + " \"type\": \"record\",\n" + + " \"namespace\": \"org.apache.tajo\",\n" + + " \"name\": \"testNullHandlingTypes\",\n" + + " \"fields\": [\n" + + " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" + + " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" + + " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" + + " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" + + " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" + + " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" + + " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" + + " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" + + " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" + + " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" + + " { \"name\": \"col11\", \"type\": \"null\" },\n" + + " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" + + " ]\n" + + "}\n"; private static String TEST_MAX_VALUE_AVRO_SCHEMA = "{\n" + @@ -112,18 +113,20 @@ public class TestStorages { private boolean splitable; private boolean statsable; private boolean seekable; + private boolean internalType; private Path testDir; private FileSystem fs; - public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable) throws IOException { + public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable, boolean internalType) + throws IOException { this.storeType = type; this.splitable = splitable; this.statsable = statsable; this.seekable = seekable; - + this.internalType = internalType; conf = new TajoConf(); - if (storeType.equalsIgnoreCase("RCFILE")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100); } @@ -134,18 +137,19 @@ public TestStorages(String type, boolean splitable, boolean statsable, boolean s @Parameterized.Parameters public static Collection generateParameters() { return Arrays.asList(new Object[][] { - //type, splitable, statsable, seekable - {"RAW", false, true, true}, - {"RCFILE", true, true, false}, - {"PARQUET", false, false, false}, - {"SEQUENCEFILE", true, true, false}, - {"AVRO", false, false, false}, - {"TEXT", true, true, true}, - {"JSON", true, true, false}, + //type, splitable, statsable, seekable, internalType + {BuiltinStorages.RAW, false, true, true, true}, + {BuiltinStorages.DRAW, false, true, false, true}, + {BuiltinStorages.RCFILE, true, true, false, false}, + {BuiltinStorages.PARQUET, false, false, false, false}, + {BuiltinStorages.SEQUENCE_FILE, true, true, false, false}, + {BuiltinStorages.AVRO, false, false, false, false}, + {BuiltinStorages.TEXT, true, true, true, false}, + {BuiltinStorages.JSON, true, true, false, false}, }); } - @Test + @Test public void testSplitable() throws IOException { if (splitable) { Schema schema = new Schema(); @@ -198,11 +202,11 @@ public void testSplitable() throws IOException { assertEquals(tupleNum, tupleCnt); } - } + } @Test public void testRCFileSplitable() throws IOException { - if (storeType.equalsIgnoreCase("StoreType.RCFILE")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.INT8); @@ -264,9 +268,9 @@ public void testProjection() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(storeType); meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase("AVRO")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, - TEST_PROJECTION_AVRO_SCHEMA); + TEST_PROJECTION_AVRO_SCHEMA); } Path tablePath = new Path(testDir, "testProjection.data"); @@ -309,8 +313,8 @@ private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCn assertTrue(tupleCnt + 2 == tuple.getInt8(0)); assertTrue(tupleCnt + 3 == tuple.getFloat4(1)); } else { - // RAW and ROW always project all fields. - if (!storeType.equalsIgnoreCase("RAW") && !storeType.equalsIgnoreCase("ROWFILE")) { + // Internal storage always project all fields. + if (!internalType) { assertTrue(tuple.isBlankOrNull(0)); } assertTrue(tupleCnt + 2 == tuple.getInt8(1)); @@ -320,7 +324,7 @@ private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCn @Test public void testVariousTypes() throws IOException { - boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON"); + boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -341,7 +345,7 @@ public void testVariousTypes() throws IOException { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase("AVRO")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString(); meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path); } @@ -393,7 +397,7 @@ public void testVariousTypes() throws IOException { @Test public void testNullHandlingTypes() throws IOException { - boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON"); + boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -490,7 +494,7 @@ public void testNullHandlingTypes() throws IOException { @Test public void testRCFileTextSerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("RCFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -559,7 +563,7 @@ public void testRCFileTextSerializeDeserialize() throws IOException { @Test public void testRCFileBinarySerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("RCFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -628,7 +632,7 @@ public void testRCFileBinarySerializeDeserialize() throws IOException { @Test public void testSequenceFileTextSerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -701,7 +705,7 @@ public void testSequenceFileTextSerializeDeserialize() throws IOException { @Test public void testSequenceFileBinarySerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return; + if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -775,7 +779,7 @@ public void testSequenceFileBinarySerializeDeserialize() throws IOException { @Test public void testTime() throws IOException { - if (storeType.equalsIgnoreCase("TEXT") || storeType.equalsIgnoreCase("RAW")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT) || internalType) { Schema schema = new Schema(); schema.addColumn("col1", Type.DATE); schema.addColumn("col2", Type.TIME); @@ -869,7 +873,7 @@ public void testSeekableScanner() throws IOException { long readRows = 0; for (long offset : offsets) { scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, - new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema); + new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema); scanner.init(); while (scanner.next() != null) { @@ -903,11 +907,11 @@ public void testMaxValue() throws IOException { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - if (storeType.equalsIgnoreCase("AVRO")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA); } - if (storeType.equalsIgnoreCase("RAW")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.RAW)) { OldStorageManager.clearCache(); /* TAJO-1250 reproduce BufferOverflow of RAWFile */ int headerSize = 4 + 2 + 1; //Integer record length + Short null-flag length + 1 byte null flags @@ -948,16 +952,16 @@ int record = 4 + 8 + 2 + 5 + 8; // required size is 27 scanner.close(); - if (storeType.equalsIgnoreCase("RAW")){ + if (internalType){ OldStorageManager.clearCache(); } } @Test public void testLessThanSchemaSize() throws IOException { - /* RAW is internal storage. It must be same with schema size */ - if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO") - || storeType.equalsIgnoreCase("PARQUET")){ + /* Internal storage must be same with schema size */ + if (internalType || storeType.equalsIgnoreCase(BuiltinStorages.AVRO) + || storeType.equalsIgnoreCase(BuiltinStorages.PARQUET)) { return; } @@ -1020,10 +1024,10 @@ public void testLessThanSchemaSize() throws IOException { @Test public final void testInsertFixedCharTypeWithOverSize() throws Exception { - if (storeType.equalsIgnoreCase("TEXT") == false && - storeType.equalsIgnoreCase("SEQUENCEFILE") == false && - storeType.equalsIgnoreCase("RCFILE") == false && - storeType.equalsIgnoreCase("PARQUET") == false) { + if (!storeType.equalsIgnoreCase(BuiltinStorages.TEXT) && + !storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE) && + !storeType.equalsIgnoreCase(BuiltinStorages.RCFILE) && + !storeType.equalsIgnoreCase(BuiltinStorages.PARQUET)) { return; } @@ -1063,4 +1067,4 @@ public final void testInsertFixedCharTypeWithOverSize() throws Exception { assertTrue(ok); } -} +} \ No newline at end of file diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java index 46c0d6e7c3..5a5774844f 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java @@ -30,6 +30,7 @@ import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; @@ -40,8 +41,8 @@ import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.rawfile.DirectRawFileScanner; import org.apache.tajo.storage.rawfile.DirectRawFileWriter; -import org.apache.tajo.tuple.offheap.OffHeapRowBlock; -import org.apache.tajo.tuple.offheap.RowWriter; +import org.apache.tajo.tuple.memory.MemoryRowBlock; +import org.apache.tajo.tuple.memory.RowWriter; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.ProtoUtil; @@ -147,7 +148,7 @@ public Path getTestDir(FileSystem fs, String dir) throws IOException { CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, PrimitiveProtos.StringProto.class.getName())); } - public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock rowBlock, Path outputFile) + public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock, Path outputFile) throws IOException { DirectRawFileWriter writer = new DirectRawFileWriter(conf, null, schema, meta, outputFile); writer.init(); @@ -160,7 +161,7 @@ public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock r return status; } - public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock rowBlock) throws IOException { + public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock) throws IOException { Path outputDir = new Path(testDir, UUID.randomUUID() + ""); outputDir.getFileSystem(conf).mkdirs(outputDir); Path outputFile = new Path(outputDir, "output.draw"); @@ -171,7 +172,7 @@ public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock r public void testRWForAllTypesWithNextTuple() throws IOException { int rowNum = 10000; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW); FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock); @@ -201,7 +202,7 @@ public void testRWForAllTypesWithNextTuple() throws IOException { public void testRepeatedScan() throws IOException { int rowNum = 2; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW); FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock); @@ -229,7 +230,7 @@ public void testRepeatedScan() throws IOException { public void testReset() throws IOException { int rowNum = 2; - OffHeapRowBlock rowBlock = createRowBlock(rowNum); + MemoryRowBlock rowBlock = createRowBlock(rowNum); TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW); FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock); @@ -264,11 +265,11 @@ public void testReset() throws IOException { reader.close(); } - public static OffHeapRowBlock createRowBlock(int rowNum) { + public static MemoryRowBlock createRowBlock(int rowNum) { long allocateStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 8); + MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), StorageUnit.MB * 8); long allocatedEnd = System.currentTimeMillis(); - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " + LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (allocatedEnd - allocateStart) + " msec"); long writeStart = System.currentTimeMillis(); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml deleted file mode 100644 index 2de1617ee3..0000000000 --- a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml +++ /dev/null @@ -1,186 +0,0 @@ - - - - - - - - fs.s3.impl - org.apache.tajo.storage.s3.SmallBlockS3FileSystem - - - - - tajo.storage.manager.hdfs.class - org.apache.tajo.storage.FileTablespace - - - tajo.storage.manager.hbase.class - org.apache.tajo.storage.hbase.HBaseTablespace - - - - - tajo.storage.scanner-handler - text,json,raw,rcfile,row,parquet,sequencefile,avro - - - - - tajo.storage.fragment.text.class - org.apache.tajo.storage.fragment.FileFragment - - - tajo.storage.fragment.json.class - org.apache.tajo.storage.fragment.FileFragment - - - tajo.storage.fragment.raw.class - org.apache.tajo.storage.fragment.FileFragment - - - tajo.storage.fragment.rcfile.class - org.apache.tajo.storage.fragment.FileFragment - - - tajo.storage.fragment.row.class - org.apache.tajo.storage.fragment.FileFragment - - - tajo.storage.fragment.parquet.class - org.apache.tajo.storage.fragment.FileFragment - - - tajo.storage.fragment.sequencefile.class - org.apache.tajo.storage.fragment.FileFragment - - - tajo.storage.fragment.avro.class - org.apache.tajo.storage.fragment.FileFragment - - - - - tajo.storage.scanner-handler.text.class - org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner - - - - tajo.storage.scanner-handler.json.class - org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner - - - - tajo.storage.scanner-handler.raw.class - org.apache.tajo.storage.RawFile$RawFileScanner - - - - tajo.storage.scanner-handler.rcfile.class - org.apache.tajo.storage.rcfile.RCFile$RCFileScanner - - - - tajo.storage.scanner-handler.rowfile.class - org.apache.tajo.storage.RowFile$RowFileScanner - - - - tajo.storage.scanner-handler.parquet.class - org.apache.tajo.storage.parquet.ParquetScanner - - - - tajo.storage.scanner-handler.sequencefile.class - org.apache.tajo.storage.sequencefile.SequenceFileScanner - - - - tajo.storage.scanner-handler.avro.class - org.apache.tajo.storage.avro.AvroScanner - - - - - tajo.storage.appender-handler - text,raw,rcfile,row,parquet,sequencefile,avro - - - - tajo.storage.appender-handler.text.class - org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender - - - - tajo.storage.appender-handler.json.class - org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender - - - - tajo.storage.appender-handler.raw.class - org.apache.tajo.storage.RawFile$RawFileAppender - - - - tajo.storage.appender-handler.rcfile.class - org.apache.tajo.storage.rcfile.RCFile$RCFileAppender - - - - tajo.storage.appender-handler.rowfile.class - org.apache.tajo.storage.RowFile$RowFileAppender - - - - tajo.storage.appender-handler.parquet.class - org.apache.tajo.storage.parquet.ParquetAppender - - - - tajo.storage.appender-handler.sequencefile.class - org.apache.tajo.storage.sequencefile.SequenceFileAppender - - - - tajo.storage.appender-handler.avro.class - org.apache.tajo.storage.avro.AvroAppender - - - - - tajo.storage.text.io.read-buffer.bytes - 131072 - 128KB read buffer - - - tajo.storage.text.io.write-buffer.bytes - 131072 - 128KB write buffer - - - tajo.storage.raw.io.read-buffer.bytes - 131072 - 128KB read buffer - - - tajo.storage.raw.io.write-buffer.bytes - 131072 - 128KB write buffer - - From 57c2a55e13de4ec36d3f9301dbc999f2ed6d4092 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Thu, 27 Aug 2015 20:43:06 +0900 Subject: [PATCH 02/15] fix broken test --- .../java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java index 5ace4464ba..2580aaf2ea 100644 --- a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java @@ -24,6 +24,7 @@ import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.ProtobufDatum; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.unit.StorageUnit; @@ -59,7 +60,7 @@ public class TestMemoryRowBlock { DataType.newBuilder().setType(Type.TIME).build(), DataType.newBuilder().setType(Type.INTERVAL).build(), DataType.newBuilder().setType(Type.INET4).build(), - DataType.newBuilder().setType(Type.PROTOBUF).build() + DataType.newBuilder().setType(Type.PROTOBUF).setCode(PrimitiveProtos.StringProto.class.getName()).build() }; } From 617583b33a75e0b9606d2eeac3e758a18d367326 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Sun, 30 Aug 2015 21:17:01 +0900 Subject: [PATCH 03/15] TAJO-1738: Improve off-heap RowBlock --- .../org/apache/tajo/storage/BufferPool.java | 20 ++- .../apache/tajo/tuple/BaseTupleBuilder.java | 18 +-- .../tajo/tuple/memory/DirectBufTuple.java | 2 +- .../apache/tajo/tuple/memory/HeapTuple.java | 99 ++++++++------- .../apache/tajo/tuple/memory/MemoryBlock.java | 16 ++- .../tajo/tuple/memory/MemoryRowBlock.java | 46 ++++--- .../tuple/memory/OffHeapRowBlockReader.java | 29 +++-- .../tuple/memory/OffHeapRowBlockUtils.java | 36 +++++- .../tuple/memory/OffHeapRowBlockWriter.java | 5 + .../tajo/tuple/memory/OffHeapRowWriter.java | 26 ++-- ...ryBlock.java => ResizableMemoryBlock.java} | 37 +++--- .../apache/tajo/tuple/memory/UnSafeTuple.java | 117 +++++++++--------- .../tajo/tuple/memory/ZeroCopyTuple.java | 25 +++- .../tajo/tuple/TestBaseTupleBuilder.java | 10 +- .../tajo/tuple/memory/TestHeapTuple.java | 48 ++++++- .../tajo/tuple/memory/TestMemoryRowBlock.java | 44 ++++--- .../storage/rawfile/DirectRawFileScanner.java | 6 +- .../storage/rawfile/DirectRawFileWriter.java | 16 +-- 18 files changed, 369 insertions(+), 231 deletions(-) rename tajo-common/src/main/java/org/apache/tajo/tuple/memory/{OffHeapMemoryBlock.java => ResizableMemoryBlock.java} (84%) diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java index be298fc0f8..3225556371 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java @@ -18,10 +18,7 @@ package org.apache.tajo.storage; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.PooledByteBufAllocator; -import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.buffer.*; import io.netty.util.ResourceLeakDetector; import io.netty.util.internal.PlatformDependent; import org.apache.hadoop.classification.InterfaceStability; @@ -29,6 +26,7 @@ import org.apache.tajo.util.CommonTestingUtil; import java.lang.reflect.Field; +import java.nio.ByteOrder; /* this class is PooledBuffer holder */ public class BufferPool { @@ -106,7 +104,17 @@ public static ByteBuf directBuffer(int size) { * @return allocated ByteBuf from pool */ public static ByteBuf directBuffer(int size, int max) { - return ALLOCATOR.directBuffer(size, max); + return ALLOCATOR.directBuffer(size, max).order(ByteOrder.nativeOrder()); + } + + /** + * + * @param size the initial capacity + * @param max the max capacity + * @return heap ByteBuf + */ + public static ByteBuf heapBuffer(int size, int max) { + return Unpooled.buffer(size, max).order(ByteOrder.nativeOrder()); } @InterfaceStability.Unstable @@ -120,6 +128,6 @@ public static void forceRelease(ByteBuf buf) { * @param minWritableBytes required minimum writable size */ public static ByteBuf ensureWritable(ByteBuf buf, int minWritableBytes) { - return buf.ensureWritable(minWritableBytes); + return buf.ensureWritable(minWritableBytes).order(ByteOrder.nativeOrder()); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java b/tajo-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java index 4de4dd22b5..04a02673c6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java @@ -18,9 +18,6 @@ package org.apache.tajo.tuple; -import io.netty.util.internal.PlatformDependent; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.storage.Tuple; import org.apache.tajo.tuple.memory.*; @@ -28,14 +25,12 @@ import org.apache.tajo.util.Deallocatable; public class BaseTupleBuilder extends OffHeapRowWriter implements TupleBuilder, Deallocatable { - private static final Log LOG = LogFactory.getLog(BaseTupleBuilder.class); - // buffer private MemoryBlock memoryBlock; public BaseTupleBuilder(DataType[] schema) { super(schema); - this.memoryBlock = new OffHeapMemoryBlock(new ResizableLimitSpec(64 * StorageUnit.KB)); + this.memoryBlock = new ResizableMemoryBlock(new ResizableLimitSpec(64 * StorageUnit.KB), true); } @Override @@ -59,6 +54,7 @@ public void forward(int length) { @Override public boolean startRow() { + memoryBlock.writerPosition(0); return super.startRow(); } @@ -73,16 +69,12 @@ public Tuple build() { } public HeapTuple buildToHeapTuple() { - byte[] bytes = new byte[memoryBlock.readableBytes()]; - PlatformDependent.copyMemory(memoryBlock.address(), bytes, memoryBlock.readerPosition(), bytes.length); - memoryBlock.writerPosition(0); - return new HeapTuple(bytes, dataTypes()); + return buildToZeroCopyTuple().toHeapTuple(); } - public ZeroCopyTuple buildToZeroCopyTuple() { - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); + public UnSafeTuple buildToZeroCopyTuple() { + UnSafeTuple zcTuple = new UnSafeTuple(); zcTuple.set(memoryBlock, memoryBlock.readerPosition(), memoryBlock.readableBytes(), dataTypes()); - memoryBlock.writerPosition(0); return zcTuple; } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java index 4d5b0a93f0..68889df114 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java @@ -30,7 +30,7 @@ public class DirectBufTuple extends UnSafeTuple implements Deallocatable { public DirectBufTuple(int length, DataType[] types) { ByteBuffer bb = ByteBuffer.allocateDirect(length).order(ByteOrder.nativeOrder()); - memoryBlock = new OffHeapMemoryBlock(bb, new FixedSizeLimitSpec(length)); + memoryBlock = new ResizableMemoryBlock(bb, new FixedSizeLimitSpec(length)); set(memoryBlock, 0, length, types); } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java index c608a6d86a..6f90042388 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java @@ -20,6 +20,8 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; import org.apache.tajo.exception.TajoRuntimeException; @@ -28,30 +30,33 @@ import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.SizeOf; import org.apache.tajo.util.StringUtils; -import org.apache.tajo.util.UnsafeUtil; import org.apache.tajo.util.datetime.TimeMeta; -import sun.misc.Unsafe; -import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.Charset; import static org.apache.tajo.common.TajoDataTypes.DataType; -public class HeapTuple implements Tuple, Cloneable { - private static final Unsafe UNSAFE = UnsafeUtil.unsafe; - private static final long BASE_OFFSET = UnsafeUtil.ARRAY_BYTE_BASE_OFFSET; +public class HeapTuple extends ZeroCopyTuple implements Cloneable { + private ByteBuf buffer; + private DataType[] types; - private final byte [] data; - private final DataType [] types; + @Override + public void set(MemoryBlock memoryBlock, int relativePos, int length, DataType[] types) { + this.buffer = memoryBlock.getBuffer(); + this.types = types; + super.set(relativePos, length); + } - public HeapTuple(final byte [] bytes, final DataType [] types) { - this.data = bytes; + protected void set(final byte[] bytes, final DataType[] types) { + this.buffer = Unpooled.wrappedBuffer(bytes).order(ByteOrder.nativeOrder()); this.types = types; + super.set(0, bytes.length); } @Override public int size() { - return data.length; + return types.length; } @Override @@ -61,19 +66,15 @@ public TajoDataTypes.Type type(int fieldId) { @Override public int size(int fieldId) { - return UNSAFE.getInt(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getInt(checkNullAndGetOffset(fieldId)); } @Override public void clearOffset() { } - public ByteBuffer nioBuffer() { - return ByteBuffer.wrap(data); - } - private int getFieldOffset(int fieldId) { - return UNSAFE.getInt(data, BASE_OFFSET + (SizeOf.SIZE_OF_INT + (fieldId * SizeOf.SIZE_OF_INT))); + return buffer.getInt(getRelativePos() + SizeOf.SIZE_OF_INT + (fieldId * SizeOf.SIZE_OF_INT)); } private int checkNullAndGetOffset(int fieldId) { @@ -81,7 +82,7 @@ private int checkNullAndGetOffset(int fieldId) { if (offset == MemoryRowBlock.NULL_FIELD_OFFSET) { throw new RuntimeException("Invalid Field Access: " + fieldId); } - return offset; + return offset + getRelativePos(); } @Override @@ -128,19 +129,25 @@ public Datum asDatum(int fieldId) { switch (types[fieldId].getType()) { case BOOLEAN: return DatumFactory.createBool(getBool(fieldId)); + case BIT: + return DatumFactory.createBit(getByte(fieldId)); case INT1: case INT2: return DatumFactory.createInt2(getInt2(fieldId)); case INT4: return DatumFactory.createInt4(getInt4(fieldId)); case INT8: - return DatumFactory.createInt8(getInt4(fieldId)); + return DatumFactory.createInt8(getInt8(fieldId)); case FLOAT4: return DatumFactory.createFloat4(getFloat4(fieldId)); case FLOAT8: return DatumFactory.createFloat8(getFloat8(fieldId)); + case CHAR: + return DatumFactory.createChar(getBytes(fieldId)); case TEXT: - return DatumFactory.createText(getText(fieldId)); + return DatumFactory.createText(getBytes(fieldId)); + case BLOB : + return DatumFactory.createBlob(getBytes(fieldId)); case TIMESTAMP: return DatumFactory.createTimestamp(getInt8(fieldId)); case DATE: @@ -153,6 +160,8 @@ public Datum asDatum(int fieldId) { return DatumFactory.createInet4(getInt4(fieldId)); case PROTOBUF: return getProtobufDatum(fieldId); + case NULL_TYPE: + return NullDatum.get(); default: throw new TajoRuntimeException(new UnsupportedException("data type '" + types[fieldId] + "'")); } @@ -169,27 +178,26 @@ public long getOffset() { @Override public boolean getBool(int fieldId) { - return UNSAFE.getByte(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)) == 0x01; + return buffer.getByte(checkNullAndGetOffset(fieldId)) == 0x01; } @Override public byte getByte(int fieldId) { - return UNSAFE.getByte(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getByte(checkNullAndGetOffset(fieldId)); } @Override public char getChar(int fieldId) { - return UNSAFE.getChar(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getChar(checkNullAndGetOffset(fieldId)); } @Override public byte[] getBytes(int fieldId) { - long pos = checkNullAndGetOffset(fieldId); - int len = UNSAFE.getInt(data, BASE_OFFSET + pos); - pos += SizeOf.SIZE_OF_INT; + int pos = checkNullAndGetOffset(fieldId); + int len = buffer.getInt(pos); byte [] bytes = new byte[len]; - UNSAFE.copyMemory(data, BASE_OFFSET + pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len); + buffer.getBytes(pos + SizeOf.SIZE_OF_INT, bytes); return bytes; } @@ -200,27 +208,27 @@ public byte[] getTextBytes(int fieldId) { @Override public short getInt2(int fieldId) { - return UNSAFE.getShort(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getShort(checkNullAndGetOffset(fieldId)); } @Override public int getInt4(int fieldId) { - return UNSAFE.getInt(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getInt(checkNullAndGetOffset(fieldId)); } @Override public long getInt8(int fieldId) { - return UNSAFE.getLong(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getLong(checkNullAndGetOffset(fieldId)); } @Override public float getFloat4(int fieldId) { - return UNSAFE.getFloat(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getFloat(checkNullAndGetOffset(fieldId)); } @Override public double getFloat8(int fieldId) { - return UNSAFE.getDouble(data, BASE_OFFSET + checkNullAndGetOffset(fieldId)); + return buffer.getDouble(checkNullAndGetOffset(fieldId)); } @Override @@ -234,10 +242,9 @@ public TimeMeta getTimeDate(int fieldId) { } public IntervalDatum getInterval(int fieldId) { - long pos = checkNullAndGetOffset(fieldId); - int months = UNSAFE.getInt(data, BASE_OFFSET + pos); - pos += SizeOf.SIZE_OF_INT; - long millisecs = UNSAFE.getLong(data, BASE_OFFSET + pos); + int pos = checkNullAndGetOffset(fieldId); + int months = buffer.getInt(pos); + long millisecs = buffer.getLong(pos + SizeOf.SIZE_OF_INT); return new IntervalDatum(months, millisecs); } @@ -258,20 +265,14 @@ public Datum getProtobufDatum(int fieldId) { @Override public char[] getUnicodeChars(int fieldId) { - long pos = checkNullAndGetOffset(fieldId); - int len = UNSAFE.getInt(data, BASE_OFFSET + pos); - pos += SizeOf.SIZE_OF_INT; + int pos = checkNullAndGetOffset(fieldId); + int len = buffer.getInt(pos); byte [] bytes = new byte[len]; - UNSAFE.copyMemory(data, BASE_OFFSET + pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len); + buffer.getBytes(pos + SizeOf.SIZE_OF_INT, bytes); return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8")); } - @Override - public Tuple clone() throws CloneNotSupportedException { - return (HeapTuple) super.clone(); - } - @Override public Datum[] getValues() { Datum [] datums = new Datum[size()]; @@ -289,4 +290,12 @@ public Datum[] getValues() { public String toString() { return VTuple.toDisplayString(getValues()); } + + @Override + public Tuple clone() throws CloneNotSupportedException { + HeapTuple heapTuple = (HeapTuple) super.clone(); + heapTuple.buffer = buffer.copy(getRelativePos(), getLength()); + heapTuple.relativePos = 0; + return heapTuple; + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java index 918ee472ac..cc4536dc38 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryBlock.java @@ -18,6 +18,7 @@ package org.apache.tajo.tuple.memory; +import io.netty.buffer.ByteBuf; import org.apache.tajo.util.Deallocatable; import java.io.IOException; @@ -108,7 +109,7 @@ public interface MemoryBlock extends Deallocatable { int writeBytes(ScatteringByteChannel in) throws IOException; /** - * Transfers the content of this bufer to the byte array + * Transfers the content of this buffer to the byte array * @param dst the destination byte array * @param dstIndex the first index of the destination * @param length the number of bytes to transfer @@ -116,6 +117,12 @@ public interface MemoryBlock extends Deallocatable { */ int getBytes(byte[] dst, int dstIndex, int length) throws IOException; + /** + * This method does not modify {@code readerPosition} or {@code writerPosition} of this buffer. + * @return a 32-bit integer in this buffer + */ + int getInt(int index); + /** * Transfers the content of this buffer to the channel * @param out the output channel @@ -137,7 +144,12 @@ public interface MemoryBlock extends Deallocatable { int writeTo(OutputStream out) throws IOException; /** - * @return a {@MemoryBlock} which shares the whole region of this. + * @return a MemoryBlock which shares the whole region of this. */ MemoryBlock duplicate(); + + /** + * @return a internal buffer + */ + ByteBuf getBuffer(); } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java index 493a2d8d14..d297b90731 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java @@ -18,10 +18,9 @@ package org.apache.tajo.tuple.memory; -import com.google.common.annotations.VisibleForTesting; import io.netty.util.internal.PlatformDependent; import org.apache.tajo.exception.NotImplementedException; -import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.tuple.RowBlockReader; import org.apache.tajo.util.Deallocatable; import org.apache.tajo.util.SizeOf; @@ -44,28 +43,30 @@ public class MemoryRowBlock implements RowBlock, Deallocatable { private RowWriter builder; private MemoryBlock memory; - public MemoryRowBlock(DataType[] dataTypes, ResizableLimitSpec limitSpec, boolean offheap) { - if (offheap) { - this.memory = new OffHeapMemoryBlock(limitSpec); - } else { - throw new TajoRuntimeException(new NotImplementedException("Heap memory not implemented yet")); - } + public MemoryRowBlock(DataType[] dataTypes, ResizableLimitSpec limitSpec, boolean isDirect) { + this.memory = new ResizableMemoryBlock(limitSpec, isDirect); this.dataTypes = dataTypes; - this.builder = new OffHeapRowBlockWriter(this); } public MemoryRowBlock(MemoryRowBlock rowBlock) { - this.memory = TUtil.checkTypeAndGet(rowBlock.getMemory().duplicate(), OffHeapMemoryBlock.class); + this.memory = TUtil.checkTypeAndGet(rowBlock.getMemory().duplicate(), ResizableMemoryBlock.class); this.rowNum = rowBlock.rowNum; this.dataTypes = rowBlock.dataTypes; - this.builder = new OffHeapRowBlockWriter(this); } - @VisibleForTesting + public MemoryRowBlock(MemoryBlock memory, DataType[] dataTypes, int rowNum) { + this.memory = memory; + this.rowNum = rowNum; + this.dataTypes = dataTypes; + } + public MemoryRowBlock(DataType[] dataTypes, int bytes) { this(dataTypes, new ResizableLimitSpec(bytes), true); } + public MemoryRowBlock(DataType[] dataTypes, int bytes, boolean isDirect) { + this(dataTypes, new ResizableLimitSpec(bytes), isDirect); + } @Override public void clear() { @@ -75,7 +76,9 @@ public void clear() { private void reset() { rowNum = 0; - builder.clear(); + if (builder != null) { + builder.clear(); + } } @Override @@ -103,8 +106,7 @@ public DataType[] getDataTypes() { } @Override - public boolean copyFromChannel(ScatteringByteChannel channel) - throws IOException { + public boolean copyFromChannel(ScatteringByteChannel channel) throws IOException { reset(); int readBytes = memory.writeBytes(channel); @@ -135,6 +137,14 @@ public boolean copyFromChannel(ScatteringByteChannel channel) @Override public RowWriter getWriter() { + + if (builder == null) { + if (!getMemory().hasAddress()) { + throw new TajoInternalError(new NotImplementedException("Heap memory writer not implemented yet")); + } else { + this.builder = new OffHeapRowBlockWriter(this); + } + } return builder; } @@ -150,6 +160,10 @@ public void release() { @Override public RowBlockReader getReader() { - return new OffHeapRowBlockReader(this); + if (!getMemory().hasAddress()) { + return new HeapRowBlockReader(this); + } else { + return new OffHeapRowBlockReader(this); + } } } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockReader.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockReader.java index 16f5b492ea..ccaeffc217 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockReader.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockReader.java @@ -19,34 +19,48 @@ package org.apache.tajo.tuple.memory; import io.netty.util.internal.PlatformDependent; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.tuple.RowBlockReader; public class OffHeapRowBlockReader implements RowBlockReader { - private MemoryRowBlock rowBlock; + private final DataType[] dataTypes; + private final MemoryBlock memoryBlock; + private final int rows; // Read States private int curRowIdxForRead; private int curPosForRead; public OffHeapRowBlockReader(MemoryRowBlock rowBlock) { - this.rowBlock = rowBlock; + this(rowBlock.getMemory(), rowBlock.getDataTypes(), rowBlock.rows()); + } + + public OffHeapRowBlockReader(MemoryBlock memoryBlock, DataType[] dataTypes, int rows) { + this.memoryBlock = memoryBlock; + this.dataTypes = dataTypes; + this.rows = rows; + if (!memoryBlock.hasAddress()) { + throw new TajoInternalError(memoryBlock.getClass().getSimpleName() + + " does not support to direct memory access"); + } } public long remainForRead() { - return rowBlock.capacity() - curPosForRead; + return memoryBlock.readableBytes(); } @Override public boolean next(ZeroCopyTuple tuple) { - if (curRowIdxForRead < rowBlock.rows()) { + if (curRowIdxForRead < rows) { - long recordStartPtr = rowBlock.getMemory().address() + curPosForRead; + long recordStartPtr = memoryBlock.address() + curPosForRead; int recordLen = PlatformDependent.getInt(recordStartPtr); - tuple.set(rowBlock.getMemory(), curPosForRead, recordLen, rowBlock.getDataTypes()); + tuple.set(memoryBlock, curPosForRead, recordLen, dataTypes); curPosForRead += recordLen; curRowIdxForRead++; - rowBlock.getMemory().readerPosition(curPosForRead); + memoryBlock.readerPosition(curPosForRead); return true; } else { @@ -58,5 +72,6 @@ public boolean next(ZeroCopyTuple tuple) { public void reset() { curPosForRead = 0; curRowIdxForRead = 0; + memoryBlock.readerPosition(curPosForRead); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java index f2f71738d0..e8f219cebc 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java @@ -24,6 +24,7 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.tuple.RowBlockReader; import java.util.Arrays; import java.util.Collections; @@ -34,11 +35,23 @@ public class OffHeapRowBlockUtils { public static List sort(MemoryRowBlock rowBlock, Comparator comparator) { List tupleList = Lists.newArrayList(); - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); + + ZeroCopyTuple zcTuple; + if(rowBlock.getMemory().hasAddress()) { + zcTuple = new UnSafeTuple(); + } else { + zcTuple = new HeapTuple(); + } + + RowBlockReader reader = rowBlock.getReader(); while(reader.next(zcTuple)) { tupleList.add(zcTuple); - zcTuple = new ZeroCopyTuple(); + + if(rowBlock.getMemory().hasAddress()) { + zcTuple = new UnSafeTuple(); + } else { + zcTuple = new HeapTuple(); + } } Collections.sort(tupleList, comparator); return tupleList; @@ -46,11 +59,22 @@ public static List sort(MemoryRowBlock rowBlock, Comparator compar public static Tuple[] sortToArray(MemoryRowBlock rowBlock, Comparator comparator) { Tuple[] tuples = new Tuple[rowBlock.rows()]; - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); + + ZeroCopyTuple zcTuple; + if(rowBlock.getMemory().hasAddress()) { + zcTuple = new UnSafeTuple(); + } else { + zcTuple = new HeapTuple(); + } + + RowBlockReader reader = rowBlock.getReader(); for (int i = 0; i < rowBlock.rows() && reader.next(zcTuple); i++) { tuples[i] = zcTuple; - zcTuple = new ZeroCopyTuple(); + if(rowBlock.getMemory().hasAddress()) { + zcTuple = new UnSafeTuple(); + } else { + zcTuple = new HeapTuple(); + } } Arrays.sort(tuples, comparator); return tuples; diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockWriter.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockWriter.java index 322d7ca43f..6832730395 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockWriter.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockWriter.java @@ -19,6 +19,7 @@ package org.apache.tajo.tuple.memory; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.TajoInternalError; public class OffHeapRowBlockWriter extends OffHeapRowWriter { private RowBlock rowBlock; @@ -26,6 +27,10 @@ public class OffHeapRowBlockWriter extends OffHeapRowWriter { OffHeapRowBlockWriter(RowBlock rowBlock) { super(rowBlock.getDataTypes()); this.rowBlock = rowBlock; + if (!rowBlock.getMemory().hasAddress()) { + throw new TajoInternalError(rowBlock.getMemory().getClass().getSimpleName() + + " does not support to direct memory access"); + } } public long address() { diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java index 8e50f888df..c9b233f822 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowWriter.java @@ -123,13 +123,13 @@ public boolean startRow() { public void endRow() { long rowHeaderPos = recordStartAddr(); // curOffset is equivalent to a byte length of this row. - OffHeapMemoryBlock.UNSAFE.putInt(rowHeaderPos, curOffset); + PlatformDependent.putInt(rowHeaderPos, curOffset); //forward (record offset + fields offset) rowHeaderPos += SizeOf.SIZE_OF_INT + curFieldOffset; // set remain header field length for (int i = curFieldIdx; i < dataTypes.length; i++) { - OffHeapMemoryBlock.UNSAFE.putInt(rowHeaderPos, MemoryRowBlock.NULL_FIELD_OFFSET); + PlatformDependent.putInt(rowHeaderPos, MemoryRowBlock.NULL_FIELD_OFFSET); rowHeaderPos += SizeOf.SIZE_OF_INT; } } @@ -154,7 +154,7 @@ private void putFieldHeader(long currentAddr, int length) { long currentHeaderAddr = currentAddr - curOffset + curFieldOffset; // set header field length - OffHeapMemoryBlock.UNSAFE.putInt(currentHeaderAddr, length); + PlatformDependent.putInt(currentHeaderAddr, length); curFieldOffset += SizeOf.SIZE_OF_INT; curFieldIdx++; } @@ -164,7 +164,7 @@ public void putByte(byte val) { ensureSize(SizeOf.SIZE_OF_BYTE); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putByte(addr, val); + PlatformDependent.putByte(addr, val); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_BYTE); } @@ -174,7 +174,7 @@ public void putBool(boolean val) { ensureSize(SizeOf.SIZE_OF_BOOL); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putByte(addr, (byte) (val ? 0x01 : 0x00)); + PlatformDependent.putByte(addr, (byte) (val ? 0x01 : 0x00)); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_BOOL); } @@ -184,7 +184,7 @@ public void putInt2(short val) { ensureSize(SizeOf.SIZE_OF_SHORT); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putShort(addr, val); + PlatformDependent.putShort(addr, val); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_SHORT); } @@ -194,7 +194,7 @@ public void putInt4(int val) { ensureSize(SizeOf.SIZE_OF_INT); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putInt(addr, val); + PlatformDependent.putInt(addr, val); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_INT); } @@ -204,7 +204,7 @@ public void putInt8(long val) { ensureSize(SizeOf.SIZE_OF_LONG); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putLong(addr, val); + PlatformDependent.putLong(addr, val); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_LONG); } @@ -214,7 +214,7 @@ public void putFloat4(float val) { ensureSize(SizeOf.SIZE_OF_INT); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putInt(addr, Float.floatToRawIntBits(val)); + PlatformDependent.putInt(addr, Float.floatToRawIntBits(val)); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_INT); } @@ -224,7 +224,7 @@ public void putFloat8(double val) { ensureSize(SizeOf.SIZE_OF_LONG); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putLong(addr, Double.doubleToRawLongBits(val)); + PlatformDependent.putLong(addr, Double.doubleToRawLongBits(val)); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_LONG); } @@ -248,7 +248,7 @@ public void putBlob(byte[] val) { ensureSize(fieldLen); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putInt(addr, bytesLen); + PlatformDependent.putInt(addr, bytesLen); PlatformDependent.copyMemory(val, 0, addr + SizeOf.SIZE_OF_INT, bytesLen); putFieldHeader(addr, curOffset); forwardField(fieldLen); @@ -274,8 +274,8 @@ public void putInterval(IntervalDatum val) { ensureSize(SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_LONG); long addr = currentAddr(); - OffHeapMemoryBlock.UNSAFE.putInt(addr, val.getMonths()); - OffHeapMemoryBlock.UNSAFE.putLong(addr + SizeOf.SIZE_OF_INT, val.getMilliSeconds()); + PlatformDependent.putInt(addr, val.getMonths()); + PlatformDependent.putLong(addr + SizeOf.SIZE_OF_INT, val.getMilliSeconds()); putFieldHeader(addr, curOffset); forwardField(SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_LONG); } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapMemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java similarity index 84% rename from tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapMemoryBlock.java rename to tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java index ac035ed352..5a10eb51f4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapMemoryBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java @@ -26,37 +26,36 @@ import org.apache.tajo.storage.BufferPool; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.UnsafeUtil; -import sun.misc.Unsafe; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.channels.GatheringByteChannel; import java.nio.channels.ScatteringByteChannel; -public class OffHeapMemoryBlock implements MemoryBlock { - private static final Log LOG = LogFactory.getLog(OffHeapMemoryBlock.class); +public class ResizableMemoryBlock implements MemoryBlock { + private static final Log LOG = LogFactory.getLog(ResizableMemoryBlock.class); protected ByteBuf buffer; protected ResizableLimitSpec limitSpec; - protected static final Unsafe UNSAFE = UnsafeUtil.unsafe; - protected OffHeapMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { - Preconditions.checkState(buffer.hasMemoryAddress()); + protected ResizableMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { this.buffer = buffer; this.limitSpec = limitSpec; } - protected OffHeapMemoryBlock(ByteBuffer buffer, ResizableLimitSpec limitSpec) { - Preconditions.checkState(buffer.isDirect()); + protected ResizableMemoryBlock(ByteBuffer buffer, ResizableLimitSpec limitSpec) { this.buffer = Unpooled.wrappedBuffer(buffer); this.limitSpec = limitSpec; } - public OffHeapMemoryBlock(ResizableLimitSpec limitSpec) { - this(BufferPool.directBuffer((int) limitSpec.initialSize(), - (int) limitSpec.limit()).order(ByteOrder.nativeOrder()), limitSpec); + public ResizableMemoryBlock(ResizableLimitSpec limitSpec, boolean isDirect) { + if (isDirect) { + this.buffer = BufferPool.directBuffer((int) limitSpec.initialSize(), (int) limitSpec.limit()); + } else { + this.buffer = BufferPool.heapBuffer((int) limitSpec.initialSize(), (int) limitSpec.limit()); + } + this.limitSpec = limitSpec; } @Override @@ -66,7 +65,7 @@ public long address() { @Override public boolean hasAddress() { - return true; + return buffer.hasMemoryAddress(); } @Override @@ -156,7 +155,12 @@ public void release() { @Override public MemoryBlock duplicate() { - return new OffHeapMemoryBlock(buffer.duplicate().readerIndex(0), limitSpec); + return new ResizableMemoryBlock(buffer.duplicate().readerIndex(0), limitSpec); + } + + @Override + public ByteBuf getBuffer() { + return buffer; } @Override @@ -188,6 +192,11 @@ public int getBytes(byte[] bytes, int dstIndex, int length) throws IOException { return readableBytes - buffer.readableBytes(); } + @Override + public int getInt(int index) { + return buffer.getInt(index); + } + @Override public int writeTo(GatheringByteChannel channel, int length) throws IOException { return buffer.readBytes(channel, length); diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java index 497613b038..d9fbb06080 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java @@ -39,28 +39,25 @@ import static org.apache.tajo.common.TajoDataTypes.DataType; -public abstract class UnSafeTuple implements Tuple { +public class UnSafeTuple extends ZeroCopyTuple { private static final Unsafe UNSAFE = UnsafeUtil.unsafe; private long address; - private int relativePos; - private int length; - private DataType [] types; + private DataType[] types; - protected void set(MemoryBlock memoryBlock, int relativePos, int length, DataType [] types) { + @Override + public void set(MemoryBlock memoryBlock, int relativePos, int length, DataType[] types) { Preconditions.checkArgument(memoryBlock.hasAddress()); this.address = memoryBlock.address(); - this.relativePos = relativePos; - this.length = length; this.types = types; + super.set(relativePos, length); } - void set(UnSafeTuple tuple) { + public void set(UnSafeTuple tuple) { this.address = tuple.address; - this.relativePos = tuple.relativePos; - this.length = tuple.length; this.types = tuple.types; + super.set(tuple.getRelativePos(), tuple.getLength()); } @Override @@ -79,34 +76,32 @@ public int size(int fieldId) { } public void writeTo(ByteBuffer bb) { - if (bb.remaining() < length) { + if (bb.remaining() < getLength()) { throw new IndexOutOfBoundsException("remaining length: " + bb.remaining() + ", tuple length: " + getLength()); } - if (length > 0) { + if (getLength() > 0) { if (bb.isDirect()) { - PlatformDependent.copyMemory(address(), PlatformDependent.directBufferAddress(bb) + bb.position(), length); + PlatformDependent.copyMemory(address(), PlatformDependent.directBufferAddress(bb) + bb.position(), getLength()); bb.position(bb.position() + getLength()); } else { - PlatformDependent.copyMemory(address(), bb.array(), bb.arrayOffset() + bb.position(), length); + PlatformDependent.copyMemory(address(), bb.array(), bb.arrayOffset() + bb.position(), getLength()); bb.position(bb.position() + getLength()); } } } public long address() { - return address + relativePos; - } - - public int getLength() { - return length; + return address + getRelativePos(); } public HeapTuple toHeapTuple() { - byte [] bytes = new byte[length]; - PlatformDependent.copyMemory(address(), bytes, 0, length); - return new HeapTuple(bytes, types); + HeapTuple heapTuple = new HeapTuple(); + byte [] bytes = new byte[getLength()]; + PlatformDependent.copyMemory(address(), bytes, 0, getLength()); + heapTuple.set(bytes, types); + return heapTuple; } private int getFieldOffset(int fieldId) { @@ -163,43 +158,43 @@ public Datum asDatum(int fieldId) { } switch (types[fieldId].getType()) { - case BOOLEAN: - return DatumFactory.createBool(getBool(fieldId)); - case BIT: - return DatumFactory.createBit(getByte(fieldId)); - case INT1: - case INT2: - return DatumFactory.createInt2(getInt2(fieldId)); - case INT4: - return DatumFactory.createInt4(getInt4(fieldId)); - case INT8: - return DatumFactory.createInt8(getInt8(fieldId)); - case FLOAT4: - return DatumFactory.createFloat4(getFloat4(fieldId)); - case FLOAT8: - return DatumFactory.createFloat8(getFloat8(fieldId)); - case CHAR: - return DatumFactory.createChar(getBytes(fieldId)); - case TEXT: - return DatumFactory.createText(getBytes(fieldId)); - case BLOB : - return DatumFactory.createBlob(getBytes(fieldId)); - case TIMESTAMP: - return DatumFactory.createTimestamp(getInt8(fieldId)); - case DATE: - return DatumFactory.createDate(getInt4(fieldId)); - case TIME: - return DatumFactory.createTime(getInt8(fieldId)); - case INTERVAL: - return getInterval(fieldId); - case INET4: - return DatumFactory.createInet4(getInt4(fieldId)); - case PROTOBUF: - return getProtobufDatum(fieldId); - case NULL_TYPE: - return NullDatum.get(); - default: - throw new TajoRuntimeException(new UnsupportedException("data type '" + types[fieldId] + "'")); + case BOOLEAN: + return DatumFactory.createBool(getBool(fieldId)); + case BIT: + return DatumFactory.createBit(getByte(fieldId)); + case INT1: + case INT2: + return DatumFactory.createInt2(getInt2(fieldId)); + case INT4: + return DatumFactory.createInt4(getInt4(fieldId)); + case INT8: + return DatumFactory.createInt8(getInt8(fieldId)); + case FLOAT4: + return DatumFactory.createFloat4(getFloat4(fieldId)); + case FLOAT8: + return DatumFactory.createFloat8(getFloat8(fieldId)); + case CHAR: + return DatumFactory.createChar(getBytes(fieldId)); + case TEXT: + return DatumFactory.createText(getBytes(fieldId)); + case BLOB: + return DatumFactory.createBlob(getBytes(fieldId)); + case TIMESTAMP: + return DatumFactory.createTimestamp(getInt8(fieldId)); + case DATE: + return DatumFactory.createDate(getInt4(fieldId)); + case TIME: + return DatumFactory.createTime(getInt8(fieldId)); + case INTERVAL: + return getInterval(fieldId); + case INET4: + return DatumFactory.createInet4(getInt4(fieldId)); + case PROTOBUF: + return getProtobufDatum(fieldId); + case NULL_TYPE: + return NullDatum.get(); + default: + throw new TajoRuntimeException(new UnsupportedException("data type '" + types[fieldId] + "'")); } } @@ -340,5 +335,7 @@ public String toString() { return VTuple.toDisplayString(getValues()); } - public abstract void release(); + public void release() { + + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ZeroCopyTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ZeroCopyTuple.java index 3f2e832170..1f4f57eab9 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ZeroCopyTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ZeroCopyTuple.java @@ -19,15 +19,30 @@ package org.apache.tajo.tuple.memory; import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.storage.Tuple; -public class ZeroCopyTuple extends UnSafeTuple { +public abstract class ZeroCopyTuple implements Tuple { - public void set(MemoryBlock memoryBlock, int relativePos, int length, DataType[] types) { - super.set(memoryBlock, relativePos, length, types); + protected int relativePos; + protected int length; + + public abstract void set(MemoryBlock memoryBlock, int relativePos, int length, DataType[] types); + + void set(int relativePos, int length) { + this.relativePos = relativePos; + this.length = length; + } + + public int getRelativePos() { + return relativePos; + } + + public int getLength() { + return length; } @Override - public void release() { - // nothing to do + public Tuple clone() throws CloneNotSupportedException { + return (Tuple) super.clone(); } } diff --git a/tajo-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java b/tajo-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java index 8a3e59c633..6ce1a6fa74 100644 --- a/tajo-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java @@ -30,7 +30,7 @@ public void testBuild() { MemoryRowBlock rowBlock = TestMemoryRowBlock.createRowBlock(10248); RowBlockReader reader = rowBlock.getReader(); - ZeroCopyTuple inputTuple = new ZeroCopyTuple(); + ZeroCopyTuple inputTuple = new UnSafeTuple(); HeapTuple heapTuple; ZeroCopyTuple zcTuple; @@ -38,12 +38,12 @@ public void testBuild() { while(reader.next(inputTuple)) { OffHeapRowBlockUtils.convert(inputTuple, builder); - heapTuple = builder.buildToHeapTuple(); - TestMemoryRowBlock.validateTupleResult(i, heapTuple); - zcTuple = builder.buildToZeroCopyTuple(); TestMemoryRowBlock.validateTupleResult(i, zcTuple); + heapTuple = builder.buildToHeapTuple(); + TestMemoryRowBlock.validateTupleResult(i, heapTuple); + i++; } builder.release(); @@ -57,7 +57,7 @@ public void testBuildWithNull() { MemoryRowBlock rowBlock = TestMemoryRowBlock.createRowBlockWithNull(10248); RowBlockReader reader = rowBlock.getReader(); - ZeroCopyTuple inputTuple = new ZeroCopyTuple(); + ZeroCopyTuple inputTuple = new UnSafeTuple(); HeapTuple heapTuple; ZeroCopyTuple zcTuple; diff --git a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java index ac93d07da8..0ae2140764 100644 --- a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java @@ -18,25 +18,65 @@ package org.apache.tajo.tuple.memory; +import io.netty.buffer.ByteBuf; +import org.apache.tajo.storage.BufferPool; +import org.apache.tajo.tuple.RowBlockReader; import org.junit.Test; +import static org.junit.Assert.*; + public class TestHeapTuple { @Test - public void testHeapTuple() { + public void testHeapTupleFromOffheap() { MemoryRowBlock rowBlock = TestMemoryRowBlock.createRowBlock(1024); + assertTrue(rowBlock.getMemory().getBuffer().isDirect()); + assertTrue(rowBlock.getMemory().hasAddress()); + + RowBlockReader reader = rowBlock.getReader(); + assertEquals(OffHeapRowBlockReader.class, reader.getClass()); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); + UnSafeTuple zcTuple = new UnSafeTuple(); int i = 0; while (reader.next(zcTuple)) { HeapTuple heapTuple = zcTuple.toHeapTuple(); - TestMemoryRowBlock.validateTupleResult(i, zcTuple); TestMemoryRowBlock.validateTupleResult(i, heapTuple); + TestMemoryRowBlock.validateTupleResult(i, zcTuple); + TestMemoryRowBlock.validateTupleResult(i, zcTuple.toHeapTuple()); i++; } + assertEquals(rowBlock.rows(), i); + rowBlock.release(); + } + + @Test + public void testHeapTupleFromHeap() throws CloneNotSupportedException { + MemoryRowBlock rowBlock = TestMemoryRowBlock.createRowBlock(1024); + int length = rowBlock.getMemory().writerPosition(); + //write rows to heap + ByteBuf heapBuffer = BufferPool.heapBuffer(length, length); + heapBuffer.writeBytes(rowBlock.getMemory().getBuffer()); + assertFalse(heapBuffer.isDirect()); + + ResizableMemoryBlock memoryBlock = + new ResizableMemoryBlock(heapBuffer, new ResizableLimitSpec((long) heapBuffer.capacity())); + assertFalse(memoryBlock.hasAddress()); + + + RowBlockReader reader = new HeapRowBlockReader(memoryBlock, rowBlock.getDataTypes(), rowBlock.rows()); + assertEquals(HeapRowBlockReader.class, reader.getClass()); + HeapTuple heapTuple = new HeapTuple(); + int i = 0; + while (reader.next(heapTuple)) { + + TestMemoryRowBlock.validateTupleResult(i, heapTuple); + TestMemoryRowBlock.validateTupleResult(i, heapTuple.clone()); + i++; + } + assertEquals(rowBlock.rows(), i); rowBlock.release(); + memoryBlock.release(); } } \ No newline at end of file diff --git a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java index 2580aaf2ea..83eea04770 100644 --- a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java @@ -27,6 +27,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; +import org.apache.tajo.tuple.RowBlockReader; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.NumberUtil; @@ -39,6 +40,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class TestMemoryRowBlock { @@ -78,14 +80,14 @@ public void testPutAndReadValidation() { long allocEnd = System.currentTimeMillis(); explainRowBlockAllocation(rowBlock, allocStart, allocEnd); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); + RowBlockReader reader = null; - ZeroCopyTuple tuple = new ZeroCopyTuple(); + ZeroCopyTuple tuple = new UnSafeTuple(); long writeStart = System.currentTimeMillis(); for (int i = 0; i < rowNum; i++) { fillRow(i, rowBlock.getWriter()); - reader.reset(); + reader = rowBlock.getReader(); int j = 0; while(reader.next(tuple)) { validateTupleResult(j, tuple); @@ -93,11 +95,12 @@ public void testPutAndReadValidation() { } } + assertNotNull(reader); long writeEnd = System.currentTimeMillis(); LOG.info("writing and validating take " + (writeEnd - writeStart) + " msec"); long readStart = System.currentTimeMillis(); - tuple = new ZeroCopyTuple(); + tuple = new UnSafeTuple(); int j = 0; reader.reset(); while(reader.next(tuple)) { @@ -120,14 +123,14 @@ public void testNullityValidation() { long allocEnd = System.currentTimeMillis(); explainRowBlockAllocation(rowBlock, allocStart, allocEnd); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - ZeroCopyTuple tuple = new ZeroCopyTuple(); + RowBlockReader reader = null; + ZeroCopyTuple tuple = new UnSafeTuple(); long writeStart = System.currentTimeMillis(); for (int i = 0; i < rowNum; i++) { fillRowBlockWithNull(i, rowBlock.getWriter()); - reader.reset(); + reader = rowBlock.getReader(); int j = 0; while(reader.next(tuple)) { validateNullity(j, tuple); @@ -135,11 +138,13 @@ public void testNullityValidation() { j++; } } + + assertNotNull(reader); long writeEnd = System.currentTimeMillis(); LOG.info("writing and nullity validating take " + (writeEnd - writeStart) + " msec"); long readStart = System.currentTimeMillis(); - tuple = new ZeroCopyTuple(); + tuple = new UnSafeTuple(); int j = 0; reader.reset(); while(reader.next(tuple)) { @@ -172,10 +177,10 @@ public void testEmptyRow() { long writeEnd = System.currentTimeMillis(); LOG.info("writing tooks " + (writeEnd - writeStart) + " msec"); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); + RowBlockReader reader = rowBlock.getReader(); long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); + ZeroCopyTuple tuple = new UnSafeTuple(); int j = 0; reader.reset(); while(reader.next(tuple)) { @@ -196,16 +201,15 @@ public void testSortBenchmark() { int rowNum = 1000; MemoryRowBlock rowBlock = createRowBlock(rowNum); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - List unSafeTuples = Lists.newArrayList(); long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); - reader.reset(); + ZeroCopyTuple tuple = new UnSafeTuple(); + + RowBlockReader reader = rowBlock.getReader(); while(reader.next(tuple)) { unSafeTuples.add(tuple); - tuple = new ZeroCopyTuple(); + tuple = new UnSafeTuple(); } long readEnd = System.currentTimeMillis(); LOG.info("reading takes " + (readEnd - readStart) + " msec"); @@ -297,19 +301,18 @@ public void testSerDerOfZeroCopyTuple() { MemoryRowBlock rowBlock = createRowBlock(rowNum); MemoryRowBlock restoredRowBlock = new MemoryRowBlock(rowBlock); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(restoredRowBlock); + RowBlockReader reader = restoredRowBlock.getReader(); long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); + UnSafeTuple tuple = new UnSafeTuple(); int j = 0; List copyTuples = Lists.newArrayList(); - reader.reset(); while (reader.next(tuple)) { validateTupleResult(j, tuple); - ZeroCopyTuple copyTuple = new ZeroCopyTuple(); + UnSafeTuple copyTuple = new UnSafeTuple(); copyTuple.set(tuple); copyTuples.add(copyTuple); @@ -481,7 +484,7 @@ public static void fillVTuple(int i, VTuple tuple) { public static void validateResults(MemoryRowBlock rowBlock) { long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); + ZeroCopyTuple tuple = new UnSafeTuple(); int j = 0; OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); reader.reset(); @@ -489,6 +492,7 @@ public static void validateResults(MemoryRowBlock rowBlock) { validateTupleResult(j, tuple); j++; } + assertEquals(rowBlock.rows(), j); long readEnd = System.currentTimeMillis(); LOG.info("Reading takes " + (readEnd - readStart) + " msec"); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java index bc2df79d06..6c3dfc1729 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java @@ -34,6 +34,7 @@ import org.apache.tajo.tuple.RowBlockReader; import org.apache.tajo.tuple.memory.MemoryRowBlock; import org.apache.tajo.tuple.memory.RowBlock; +import org.apache.tajo.tuple.memory.UnSafeTuple; import org.apache.tajo.tuple.memory.ZeroCopyTuple; import org.apache.tajo.unit.StorageUnit; @@ -45,13 +46,12 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner private static final Log LOG = LogFactory.getLog(DirectRawFileScanner.class); private SeekableInputChannel channel; - //private DataType[] columnTypes; private boolean eof = false; private long fileSize; private long recordCount; - private ZeroCopyTuple unSafeTuple = new ZeroCopyTuple(); + private ZeroCopyTuple unSafeTuple = new UnSafeTuple(); private RowBlock tupleBuffer; private RowBlockReader reader; @@ -62,7 +62,7 @@ public DirectRawFileScanner(Configuration conf, Schema schema, TableMeta meta, F public void init() throws IOException { initChannel(); - tupleBuffer = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), 64 * StorageUnit.KB); + tupleBuffer = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), 64 * StorageUnit.KB, true); reader = tupleBuffer.getReader(); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java index 25b921b8f5..fdbd18f99f 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java @@ -47,6 +47,7 @@ public class DirectRawFileWriter extends FileAppender { public static final String FILE_EXTENSION = "draw"; + private static final int BUFFER_SIZE = 64 * StorageUnit.KB; private static final Log LOG = LogFactory.getLog(DirectRawFileWriter.class); private FileChannel channel; @@ -95,22 +96,14 @@ public void init() throws IOException { PlannerUtil.getShuffleType(ShuffleType.NONE_SHUFFLE))); } - memoryRowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), 64 * StorageUnit.KB); + memoryRowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), BUFFER_SIZE, true); pos = 0; super.init(); } @Override public long getOffset() throws IOException { - return pos; - } - - private long getFilePosition() throws IOException { - if (isLocal) { - return channel.position(); - } else { - return fos.getPos(); - } + return pos + memoryRowBlock.getMemory().writerPosition(); } public void writeRowBlock(MemoryRowBlock rowBlock) throws IOException { @@ -137,7 +130,8 @@ public void addTuple(Tuple t) throws IOException { } memoryRowBlock.getWriter().addTuple(t); - if(memoryRowBlock.getMemory().readableBytes() > 64 * StorageUnit.KB) { + + if(memoryRowBlock.getMemory().readableBytes() >= BUFFER_SIZE) { writeRowBlock(memoryRowBlock); } } From a071c3831364d61b3a9b42db77ccb31e65223e46 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Sun, 30 Aug 2015 21:17:52 +0900 Subject: [PATCH 04/15] TAJO-1738 --- .../tajo/tuple/memory/HeapRowBlockReader.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapRowBlockReader.java diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapRowBlockReader.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapRowBlockReader.java new file mode 100644 index 0000000000..dd377cff0e --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapRowBlockReader.java @@ -0,0 +1,70 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.tuple.memory; + +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.tuple.RowBlockReader; + +public class HeapRowBlockReader implements RowBlockReader { + private final DataType[] dataTypes; + private final MemoryBlock memoryBlock; + private final int rows; + + // Read States + private int curRowIdxForRead; + private int curPosForRead; + + public HeapRowBlockReader(MemoryRowBlock rowBlock) { + this(rowBlock.getMemory(), rowBlock.getDataTypes(), rowBlock.rows()); + } + + public HeapRowBlockReader(MemoryBlock memoryBlock, DataType[] dataTypes, int rows) { + this.dataTypes = dataTypes; + this.rows = rows; + this.memoryBlock = memoryBlock.duplicate(); + } + + public long remainForRead() { + return memoryBlock.readableBytes(); + } + + @Override + public boolean next(HeapTuple tuple) { + if (curRowIdxForRead < rows) { + + int recordLen = memoryBlock.getInt(curPosForRead); + tuple.set(memoryBlock, curPosForRead, recordLen, dataTypes); + + curPosForRead += recordLen; + curRowIdxForRead++; + memoryBlock.readerPosition(curPosForRead); + + return true; + } else { + return false; + } + } + + @Override + public void reset() { + curPosForRead = 0; + curRowIdxForRead = 0; + memoryBlock.readerPosition(curPosForRead); + } +} From ec3bf8bb0a5450e4f63deb2e98303b06f70eb20f Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Mon, 31 Aug 2015 00:18:35 +0900 Subject: [PATCH 05/15] fix invalid table stat --- .../tuple/memory/ResizableMemoryBlock.java | 1 - .../storage/rawfile/DirectRawFileScanner.java | 38 +++++++++++++------ .../storage/rawfile/DirectRawFileWriter.java | 2 +- .../org/apache/tajo/storage/TestStorages.java | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java index 5a10eb51f4..80005985dc 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java @@ -150,7 +150,6 @@ private void resize(int newSize) { @Override public void release() { buffer.release(); - this.buffer = null; } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java index 6c3dfc1729..0172484c5d 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileScanner.java @@ -47,9 +47,11 @@ public class DirectRawFileScanner extends FileScanner implements SeekableScanner private SeekableInputChannel channel; - private boolean eof = false; + private boolean eos = false; private long fileSize; private long recordCount; + private long filePosition; + private long endOffset; private ZeroCopyTuple unSafeTuple = new UnSafeTuple(); private RowBlock tupleBuffer; @@ -87,44 +89,55 @@ private void initChannel() throws IOException { } channel = new LocalFileInputChannel(new FileInputStream(file)); - channel.seek(fragment.getStartKey()); fileSize = channel.size(); } else { channel = new FSDataInputChannel(fs.open(fragment.getPath())); - channel.seek(fragment.getStartKey()); fileSize = channel.size(); } + // initial set position + if (fragment.getStartKey() > 0) { + channel.seek(fragment.getStartKey()); + } + if (tableStats != null) { tableStats.setNumBytes(fileSize); } + filePosition = fragment.getStartKey(); + endOffset = fragment.getStartKey() + fragment.getLength(); if (LOG.isDebugEnabled()) { LOG.debug("RawFileScanner open:" + fragment.getPath() + ", offset :" + - fragment.getStartKey() + ", file capacity :" + fileSize); + fragment.getStartKey() + ", fragment length :" + fragment.getLength()); } } @Override public long getNextOffset() throws IOException { - return channel.position() - reader.remainForRead(); + return filePosition - reader.remainForRead(); } @Override public void seek(long offset) throws IOException { channel.seek(offset); + filePosition = channel.position(); + tupleBuffer.getMemory().clear(); fetchNeeded = true; } public boolean next(RowBlock rowblock) throws IOException { - return rowblock.copyFromChannel(channel); + long reamin = reader.remainForRead(); + boolean ret = rowblock.copyFromChannel(channel); + reader = rowblock.getReader(); + filePosition += rowblock.getMemory().writerPosition() - reamin; + return ret; } private boolean fetchNeeded = true; @Override public Tuple next() throws IOException { - if(eof) { + if(eos) { return null; } @@ -133,13 +146,15 @@ public Tuple next() throws IOException { if (!next(tupleBuffer)) { return null; } - reader.reset(); } fetchNeeded = !reader.next(unSafeTuple); if (!fetchNeeded) { recordCount++; + if(filePosition - reader.remainForRead() >= endOffset){ + eos = true; + } return unSafeTuple; } } @@ -148,8 +163,9 @@ public Tuple next() throws IOException { @Override public void reset() throws IOException { // reload initial buffer - seek(0); - eof = false; + filePosition = fragment.getStartKey(); + seek(filePosition); + eos = false; reader.reset(); } @@ -200,7 +216,7 @@ public float getProgress() { tableStats.setReadBytes(filePos); } - if(eof || channel == null) { + if(eos || channel == null) { tableStats.setReadBytes(fileSize); return 1.0f; } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java index fdbd18f99f..03642a7a24 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java @@ -116,7 +116,7 @@ public void writeRowBlock(MemoryRowBlock rowBlock) throws IOException { rowBlock.getMemory().clear(); if (enabledStats) { - stats.incrementRows(rowBlock.rows()); + stats.incrementRows(rowBlock.rows() - stats.getNumRows()); } } diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index bc9a6245d8..1051b3bca8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -139,7 +139,7 @@ public static Collection generateParameters() { return Arrays.asList(new Object[][] { //type, splitable, statsable, seekable, internalType {BuiltinStorages.RAW, false, true, true, true}, - {BuiltinStorages.DRAW, false, true, false, true}, + {BuiltinStorages.DRAW, false, true, true, true}, {BuiltinStorages.RCFILE, true, true, false, false}, {BuiltinStorages.PARQUET, false, false, false, false}, {BuiltinStorages.SEQUENCE_FILE, true, true, false, false}, From 3f91a1454e2f385b8d0c6c3a33437e081ec594d8 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Mon, 31 Aug 2015 14:02:12 +0900 Subject: [PATCH 06/15] add buffer cleaner --- .../main/java/org/apache/tajo/storage/BufferPool.java | 9 +++++++++ .../org/apache/tajo/tuple/memory/MemoryRowBlock.java | 5 +++++ .../apache/tajo/tuple/memory/ResizableMemoryBlock.java | 8 ++++++-- .../java/org/apache/tajo/tuple/memory/TestHeapTuple.java | 2 +- .../org/apache/tajo/storage/raw/TestDirectRawFile.java | 2 +- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java index 3225556371..9ba0949805 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java @@ -26,6 +26,7 @@ import org.apache.tajo.util.CommonTestingUtil; import java.lang.reflect.Field; +import java.nio.ByteBuffer; import java.nio.ByteOrder; /* this class is PooledBuffer holder */ @@ -130,4 +131,12 @@ public static void forceRelease(ByteBuf buf) { public static ByteBuf ensureWritable(ByteBuf buf, int minWritableBytes) { return buf.ensureWritable(minWritableBytes).order(ByteOrder.nativeOrder()); } + + /** + * deallocate the specified direct + * @param byteBuffer + */ + public static void free(ByteBuffer byteBuffer) { + PlatformDependent.freeDirectBuffer(byteBuffer); + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java index d297b90731..922fc689a1 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/MemoryRowBlock.java @@ -22,6 +22,7 @@ import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.tuple.RowBlockReader; +import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.Deallocatable; import org.apache.tajo.util.SizeOf; import org.apache.tajo.util.TUtil; @@ -60,6 +61,10 @@ public MemoryRowBlock(MemoryBlock memory, DataType[] dataTypes, int rowNum) { this.dataTypes = dataTypes; } + public MemoryRowBlock(DataType[] dataTypes) { + this(dataTypes, new ResizableLimitSpec(64 * StorageUnit.KB), true); + } + public MemoryRowBlock(DataType[] dataTypes, int bytes) { this(dataTypes, new ResizableLimitSpec(bytes), true); } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java index 80005985dc..e374856176 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java @@ -44,9 +44,13 @@ protected ResizableMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { this.limitSpec = limitSpec; } - protected ResizableMemoryBlock(ByteBuffer buffer, ResizableLimitSpec limitSpec) { + protected ResizableMemoryBlock(ByteBuf buffer) { + this(buffer, new ResizableLimitSpec(buffer.capacity())); + } + + protected ResizableMemoryBlock(ByteBuffer buffer) { this.buffer = Unpooled.wrappedBuffer(buffer); - this.limitSpec = limitSpec; + this.limitSpec = new ResizableLimitSpec(buffer.capacity()); } public ResizableMemoryBlock(ResizableLimitSpec limitSpec, boolean isDirect) { diff --git a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java index 0ae2140764..2b45428668 100644 --- a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestHeapTuple.java @@ -61,7 +61,7 @@ public void testHeapTupleFromHeap() throws CloneNotSupportedException { assertFalse(heapBuffer.isDirect()); ResizableMemoryBlock memoryBlock = - new ResizableMemoryBlock(heapBuffer, new ResizableLimitSpec((long) heapBuffer.capacity())); + new ResizableMemoryBlock(heapBuffer); assertFalse(memoryBlock.hasAddress()); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java index 5a5774844f..bae81a9e65 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java @@ -267,7 +267,7 @@ public void testReset() throws IOException { public static MemoryRowBlock createRowBlock(int rowNum) { long allocateStart = System.currentTimeMillis(); - MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), StorageUnit.MB * 8); + MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), StorageUnit.KB * 128); long allocatedEnd = System.currentTimeMillis(); LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated " + (allocatedEnd - allocateStart) + " msec"); From afe83393321449ae30eee38e4c1562c58f127806 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Mon, 31 Aug 2015 14:08:17 +0900 Subject: [PATCH 07/15] change to public --- .../org/apache/tajo/tuple/memory/ResizableMemoryBlock.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java index e374856176..d20efe019a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java @@ -39,16 +39,16 @@ public class ResizableMemoryBlock implements MemoryBlock { protected ByteBuf buffer; protected ResizableLimitSpec limitSpec; - protected ResizableMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { + public ResizableMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { this.buffer = buffer; this.limitSpec = limitSpec; } - protected ResizableMemoryBlock(ByteBuf buffer) { + public ResizableMemoryBlock(ByteBuf buffer) { this(buffer, new ResizableLimitSpec(buffer.capacity())); } - protected ResizableMemoryBlock(ByteBuffer buffer) { + public ResizableMemoryBlock(ByteBuffer buffer) { this.buffer = Unpooled.wrappedBuffer(buffer); this.limitSpec = new ResizableLimitSpec(buffer.capacity()); } From 3562f06ec465a37405eba603396f910de86c9062 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Mon, 31 Aug 2015 14:40:21 +0900 Subject: [PATCH 08/15] fix compilation error --- .../main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java index 68889df114..5e90639170 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java @@ -30,7 +30,7 @@ public class DirectBufTuple extends UnSafeTuple implements Deallocatable { public DirectBufTuple(int length, DataType[] types) { ByteBuffer bb = ByteBuffer.allocateDirect(length).order(ByteOrder.nativeOrder()); - memoryBlock = new ResizableMemoryBlock(bb, new FixedSizeLimitSpec(length)); + memoryBlock = new ResizableMemoryBlock(bb); set(memoryBlock, 0, length, types); } From cf1127526d1b2454e39374a774d62d7bfc860e54 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Mon, 31 Aug 2015 17:00:35 +0900 Subject: [PATCH 09/15] remove duplicated netty-all dependency --- tajo-cli/pom.xml | 4 ++++ tajo-cluster-tests/pom.xml | 20 +++++++++++++++++++ .../tuple/memory/ResizableMemoryBlock.java | 5 +++-- tajo-core-tests/pom.xml | 16 +++++++++++++++ tajo-jdbc/pom.xml | 4 ++++ tajo-storage/tajo-storage-common/pom.xml | 8 ++++++++ tajo-storage/tajo-storage-hbase/pom.xml | 10 ++++++++++ 7 files changed, 65 insertions(+), 2 deletions(-) diff --git a/tajo-cli/pom.xml b/tajo-cli/pom.xml index 83d70568bf..8de3c54b78 100644 --- a/tajo-cli/pom.xml +++ b/tajo-cli/pom.xml @@ -168,6 +168,10 @@ hadoop-mapreduce-client-core org.apache.hadoop + + netty-all + io.netty + diff --git a/tajo-cluster-tests/pom.xml b/tajo-cluster-tests/pom.xml index 2816048ae1..0810b4c5b5 100644 --- a/tajo-cluster-tests/pom.xml +++ b/tajo-cluster-tests/pom.xml @@ -214,6 +214,10 @@ com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 + + netty-all + io.netty + @@ -241,6 +245,10 @@ com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 + + netty-all + io.netty + @@ -265,12 +273,24 @@ ${hbase.version} test-jar test + + + netty-all + io.netty + + org.apache.hbase hbase-server ${hbase.version} test + + + netty-all + io.netty + + org.apache.hbase diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java index d20efe019a..b06b8fb084 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.channels.GatheringByteChannel; import java.nio.channels.ScatteringByteChannel; @@ -40,7 +41,7 @@ public class ResizableMemoryBlock implements MemoryBlock { protected ResizableLimitSpec limitSpec; public ResizableMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { - this.buffer = buffer; + this.buffer = buffer.order(ByteOrder.nativeOrder()); this.limitSpec = limitSpec; } @@ -49,7 +50,7 @@ public ResizableMemoryBlock(ByteBuf buffer) { } public ResizableMemoryBlock(ByteBuffer buffer) { - this.buffer = Unpooled.wrappedBuffer(buffer); + this.buffer = Unpooled.wrappedBuffer(buffer).order(ByteOrder.nativeOrder()); this.limitSpec = new ResizableLimitSpec(buffer.capacity()); } diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index a4c0587981..16b1fbd33c 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -226,6 +226,10 @@ com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 + + netty-all + io.netty + @@ -281,12 +285,24 @@ ${hbase.version} test-jar test + + + netty-all + io.netty + + org.apache.hbase hbase-server ${hbase.version} test + + + netty-all + io.netty + + org.apache.hbase diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index f73a4bde27..42ab9f349d 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -137,6 +137,10 @@ com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 + + netty-all + io.netty + diff --git a/tajo-storage/tajo-storage-common/pom.xml b/tajo-storage/tajo-storage-common/pom.xml index 7590c29e06..0fe8d4149f 100644 --- a/tajo-storage/tajo-storage-common/pom.xml +++ b/tajo-storage/tajo-storage-common/pom.xml @@ -222,6 +222,10 @@ limitations under the License. com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 + + netty-all + io.netty + @@ -273,6 +277,10 @@ limitations under the License. hadoop-mapreduce-client-core org.apache.hadoop + + netty-all + io.netty + diff --git a/tajo-storage/tajo-storage-hbase/pom.xml b/tajo-storage/tajo-storage-hbase/pom.xml index 892ff58663..16d417c804 100644 --- a/tajo-storage/tajo-storage-hbase/pom.xml +++ b/tajo-storage/tajo-storage-hbase/pom.xml @@ -233,6 +233,10 @@ com.sun.jersey.jersey-test-framework jersey-test-framework-grizzly2 + + netty-all + io.netty + @@ -306,6 +310,12 @@ hbase-server ${hbase.version} provided + + + netty-all + io.netty + + org.apache.hbase From 399c0948ce2d4ea53393aa19564fa60f7f58539f Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Mon, 31 Aug 2015 21:54:14 +0900 Subject: [PATCH 10/15] fix invalid text handling --- .../java/org/apache/tajo/tuple/memory/HeapTuple.java | 7 +++---- .../java/org/apache/tajo/tuple/memory/UnSafeTuple.java | 4 ++-- .../src/main/java/org/apache/tajo/util/UnsafeUtil.java | 10 ++-------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java index 6f90042388..7af3d4fac6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java @@ -33,7 +33,6 @@ import org.apache.tajo.util.datetime.TimeMeta; import java.nio.ByteOrder; -import java.nio.charset.Charset; import static org.apache.tajo.common.TajoDataTypes.DataType; @@ -203,7 +202,7 @@ public byte[] getBytes(int fieldId) { @Override public byte[] getTextBytes(int fieldId) { - return getText(fieldId).getBytes(); + return getText(fieldId).getBytes(TextDatum.DEFAULT_CHARSET); } @Override @@ -233,7 +232,7 @@ public double getFloat8(int fieldId) { @Override public String getText(int fieldId) { - return new String(getBytes(fieldId)); + return asDatum(fieldId).asChars(); } @Override @@ -270,7 +269,7 @@ public char[] getUnicodeChars(int fieldId) { byte [] bytes = new byte[len]; buffer.getBytes(pos + SizeOf.SIZE_OF_INT, bytes); - return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8")); + return StringUtils.convertBytesToChars(bytes, TextDatum.DEFAULT_CHARSET); } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java index d9fbb06080..ef6a9bd7eb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java @@ -239,7 +239,7 @@ public byte[] getBytes(int fieldId) { @Override public byte[] getTextBytes(int fieldId) { - return getBytes(fieldId); + return getText(fieldId).getBytes(TextDatum.DEFAULT_CHARSET); } @Override @@ -270,7 +270,7 @@ public double getFloat8(int fieldId) { @Override public String getText(int fieldId) { - return new String(getTextBytes(fieldId), TextDatum.DEFAULT_CHARSET); + return asDatum(fieldId).asChars(); } public IntervalDatum getInterval(int fieldId) { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java index ff6072e0ce..575e628925 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java @@ -18,7 +18,7 @@ package org.apache.tajo.util; import com.google.common.base.Preconditions; -import sun.misc.Cleaner; +import io.netty.util.internal.PlatformDependent; import sun.misc.Unsafe; import sun.nio.ch.DirectBuffer; @@ -132,12 +132,6 @@ public static void free(Deallocatable obj) { } public static void free(ByteBuffer bb) { - Preconditions.checkNotNull(bb); - Preconditions.checkState(bb.isDirect()); - - Cleaner cleaner = ((DirectBuffer) bb).cleaner(); - if (cleaner != null) { - cleaner.clean(); - } + PlatformDependent.freeDirectBuffer(bb); } } From 74d8fcd6209a7e5b22afb1dce7f585523fbecc38 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 1 Sep 2015 23:01:28 +0900 Subject: [PATCH 11/15] change to little endian --- .../src/main/java/org/apache/tajo/storage/BufferPool.java | 6 +++--- .../java/org/apache/tajo/tuple/memory/DirectBufTuple.java | 2 +- .../main/java/org/apache/tajo/tuple/memory/HeapTuple.java | 2 +- .../org/apache/tajo/tuple/memory/ResizableMemoryBlock.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java index 9ba0949805..3120083f07 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/BufferPool.java @@ -105,7 +105,7 @@ public static ByteBuf directBuffer(int size) { * @return allocated ByteBuf from pool */ public static ByteBuf directBuffer(int size, int max) { - return ALLOCATOR.directBuffer(size, max).order(ByteOrder.nativeOrder()); + return ALLOCATOR.directBuffer(size, max).order(ByteOrder.LITTLE_ENDIAN); } /** @@ -115,7 +115,7 @@ public static ByteBuf directBuffer(int size, int max) { * @return heap ByteBuf */ public static ByteBuf heapBuffer(int size, int max) { - return Unpooled.buffer(size, max).order(ByteOrder.nativeOrder()); + return Unpooled.buffer(size, max).order(ByteOrder.LITTLE_ENDIAN); } @InterfaceStability.Unstable @@ -129,7 +129,7 @@ public static void forceRelease(ByteBuf buf) { * @param minWritableBytes required minimum writable size */ public static ByteBuf ensureWritable(ByteBuf buf, int minWritableBytes) { - return buf.ensureWritable(minWritableBytes).order(ByteOrder.nativeOrder()); + return buf.ensureWritable(minWritableBytes).order(ByteOrder.LITTLE_ENDIAN); } /** diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java index 5e90639170..10e493f2e8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/DirectBufTuple.java @@ -29,7 +29,7 @@ public class DirectBufTuple extends UnSafeTuple implements Deallocatable { private MemoryBlock memoryBlock; public DirectBufTuple(int length, DataType[] types) { - ByteBuffer bb = ByteBuffer.allocateDirect(length).order(ByteOrder.nativeOrder()); + ByteBuffer bb = ByteBuffer.allocateDirect(length).order(ByteOrder.LITTLE_ENDIAN); memoryBlock = new ResizableMemoryBlock(bb); set(memoryBlock, 0, length, types); diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java index 7af3d4fac6..9dbfd250f4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java @@ -48,7 +48,7 @@ public void set(MemoryBlock memoryBlock, int relativePos, int length, DataType[] } protected void set(final byte[] bytes, final DataType[] types) { - this.buffer = Unpooled.wrappedBuffer(bytes).order(ByteOrder.nativeOrder()); + this.buffer = Unpooled.wrappedBuffer(bytes).order(ByteOrder.LITTLE_ENDIAN); this.types = types; super.set(0, bytes.length); } diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java index b06b8fb084..22c25617f4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/ResizableMemoryBlock.java @@ -41,7 +41,7 @@ public class ResizableMemoryBlock implements MemoryBlock { protected ResizableLimitSpec limitSpec; public ResizableMemoryBlock(ByteBuf buffer, ResizableLimitSpec limitSpec) { - this.buffer = buffer.order(ByteOrder.nativeOrder()); + this.buffer = buffer.order(ByteOrder.LITTLE_ENDIAN); this.limitSpec = limitSpec; } @@ -50,7 +50,7 @@ public ResizableMemoryBlock(ByteBuf buffer) { } public ResizableMemoryBlock(ByteBuffer buffer) { - this.buffer = Unpooled.wrappedBuffer(buffer).order(ByteOrder.nativeOrder()); + this.buffer = Unpooled.wrappedBuffer(buffer).order(ByteOrder.LITTLE_ENDIAN); this.limitSpec = new ResizableLimitSpec(buffer.capacity()); } From f7b55518373099e40a7fa372ad9bacf9152f1937 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 2 Sep 2015 15:52:55 +0900 Subject: [PATCH 12/15] add testing storage-default.xml --- .../src/test/resources/storage-default.xml | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml new file mode 100644 index 0000000000..b5ac9ae47e --- /dev/null +++ b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml @@ -0,0 +1,205 @@ + + + + + + + + fs.s3.impl + org.apache.tajo.storage.s3.SmallBlockS3FileSystem + + + + + tajo.storage.manager.hdfs.class + org.apache.tajo.storage.FileTablespace + + + + + tajo.storage.scanner-handler + text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro + + + + + tajo.storage.fragment.text.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.json.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.raw.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.draw.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.rcfile.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.row.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.parquet.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.orc.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.sequencefile.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.avro.class + org.apache.tajo.storage.fragment.FileFragment + + + + + tajo.storage.scanner-handler.text.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner + + + + tajo.storage.scanner-handler.json.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner + + + + tajo.storage.scanner-handler.raw.class + org.apache.tajo.storage.RawFile$RawFileScanner + + + + tajo.storage.scanner-handler.draw.class + org.apache.tajo.storage.rawfile.DirectRawFileScanner + + + + tajo.storage.scanner-handler.rcfile.class + org.apache.tajo.storage.rcfile.RCFile$RCFileScanner + + + + tajo.storage.scanner-handler.rowfile.class + org.apache.tajo.storage.RowFile$RowFileScanner + + + + tajo.storage.scanner-handler.parquet.class + org.apache.tajo.storage.parquet.ParquetScanner + + + + tajo.storage.scanner-handler.orc.class + org.apache.tajo.storage.orc.ORCScanner + + + + tajo.storage.scanner-handler.sequencefile.class + org.apache.tajo.storage.sequencefile.SequenceFileScanner + + + + tajo.storage.scanner-handler.avro.class + org.apache.tajo.storage.avro.AvroScanner + + + + + tajo.storage.appender-handler + text,raw,draw,rcfile,row,parquet,sequencefile,avro,hbase + + + + tajo.storage.appender-handler.text.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender + + + + tajo.storage.appender-handler.json.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender + + + + tajo.storage.appender-handler.raw.class + org.apache.tajo.storage.RawFile$RawFileAppender + + + + tajo.storage.appender-handler.draw.class + org.apache.tajo.storage.rawfile.DirectRawFileWriter + + + + tajo.storage.appender-handler.rcfile.class + org.apache.tajo.storage.rcfile.RCFile$RCFileAppender + + + + tajo.storage.appender-handler.rowfile.class + org.apache.tajo.storage.RowFile$RowFileAppender + + + + tajo.storage.appender-handler.parquet.class + org.apache.tajo.storage.parquet.ParquetAppender + + + + tajo.storage.appender-handler.sequencefile.class + org.apache.tajo.storage.sequencefile.SequenceFileAppender + + + + tajo.storage.appender-handler.avro.class + org.apache.tajo.storage.avro.AvroAppender + + + + + tajo.storage.text.io.read-buffer.bytes + 131072 + 128KB read buffer + + + tajo.storage.text.io.write-buffer.bytes + 131072 + 128KB write buffer + + + tajo.storage.raw.io.read-buffer.bytes + 131072 + 128KB read buffer + + + tajo.storage.raw.io.write-buffer.bytes + 131072 + 128KB write buffer + + \ No newline at end of file From ae88c7cc52729cfa132533b58badc80aa15836f7 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 2 Sep 2015 15:56:09 +0900 Subject: [PATCH 13/15] remove hbase in hdfs appender --- .../tajo-storage-hdfs/src/test/resources/storage-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml index b5ac9ae47e..b5a60fe926 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml @@ -133,7 +133,7 @@ tajo.storage.appender-handler - text,raw,draw,rcfile,row,parquet,sequencefile,avro,hbase + text,raw,draw,rcfile,row,parquet,sequencefile,avro From 25b8a834f6e145394226b1c3ff5783b6575893d2 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 2 Sep 2015 16:56:34 +0900 Subject: [PATCH 14/15] fix invalid text encoding in tests --- .../apache/tajo/tuple/memory/OffHeapRowBlockUtils.java | 2 +- .../apache/tajo/tuple/memory/TestMemoryRowBlock.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java index e8f219cebc..d909440e41 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java @@ -117,7 +117,7 @@ public static void convert(Tuple tuple, RowWriter writer) { break; case CHAR: case TEXT: - writer.putText(tuple.getBytes(i)); + writer.putText(tuple.getTextBytes(i)); break; case BLOB: writer.putBlob(tuple.getBytes(i)); diff --git a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java index 83eea04770..a6003c773d 100644 --- a/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java +++ b/tajo-common/src/test/java/org/apache/tajo/tuple/memory/TestMemoryRowBlock.java @@ -373,7 +373,7 @@ public static void fillRow(int i, RowWriter builder) { builder.putInt8(i); // 3 builder.putFloat4(i); // 4 builder.putFloat8(i); // 5 - builder.putText((UNICODE_FIELD_PREFIX + i).getBytes()); // 6 + builder.putText(UNICODE_FIELD_PREFIX + i); // 6 builder.putTimestamp(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + i); // 7 builder.putDate(DatumFactory.createDate("2014-04-16").asInt4() + i); // 8 builder.putTime(DatumFactory.createTime("08:48:00").asInt8() + i); // 9 @@ -424,7 +424,7 @@ public static void fillRowBlockWithNull(int i, RowWriter writer) { if (i % 6 == 0) { writer.skipField(); } else { - writer.putText((UNICODE_FIELD_PREFIX + i).getBytes()); // 6 + writer.putText(UNICODE_FIELD_PREFIX + i); // 6 } if (i % 7 == 0) { @@ -473,7 +473,7 @@ public static void fillVTuple(int i, VTuple tuple) { tuple.put(3, DatumFactory.createInt8(i)); tuple.put(4, DatumFactory.createFloat4(i)); tuple.put(5, DatumFactory.createFloat8(i)); - tuple.put(6, DatumFactory.createText((UNICODE_FIELD_PREFIX + i).getBytes())); + tuple.put(6, DatumFactory.createText(UNICODE_FIELD_PREFIX + i)); tuple.put(7, DatumFactory.createTimestamp(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + i)); // 7 tuple.put(8, DatumFactory.createDate(DatumFactory.createDate("2014-04-16").asInt4() + i)); // 8 tuple.put(9, DatumFactory.createTime(DatumFactory.createTime("08:48:00").asInt8() + i)); // 9 @@ -504,7 +504,7 @@ public static void validateTupleResult(int j, Tuple t) { assertEquals(j, t.getInt8(3)); assertTrue(j == t.getFloat4(4)); assertTrue(j == t.getFloat8(5)); - assertEquals(new String(UNICODE_FIELD_PREFIX + j), t.getText(6)); + assertEquals(UNICODE_FIELD_PREFIX + j, t.getText(6)); assertEquals(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + (long) j, t.getInt8(7)); assertEquals(DatumFactory.createDate("2014-04-16").asInt4() + j, t.getInt4(8)); assertEquals(DatumFactory.createTime("08:48:00").asInt8() + j, t.getInt8(9)); @@ -553,7 +553,7 @@ public static void validateNullity(int j, Tuple tuple) { if (j % 6 == 0) { tuple.isBlankOrNull(6); } else { - assertEquals(new String(UNICODE_FIELD_PREFIX + j), tuple.getText(6)); + assertEquals(UNICODE_FIELD_PREFIX + j, tuple.getText(6)); } if (j % 7 == 0) { From 5e5ed7ace92b11aeecbed8fb1840df1cf0c19bb3 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Wed, 2 Sep 2015 20:52:24 +0900 Subject: [PATCH 15/15] cleanup text converting --- .../main/java/org/apache/tajo/tuple/memory/HeapTuple.java | 4 ++-- .../org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java | 2 +- .../java/org/apache/tajo/tuple/memory/UnSafeTuple.java | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java index 9dbfd250f4..5d2fdc9079 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/HeapTuple.java @@ -202,7 +202,7 @@ public byte[] getBytes(int fieldId) { @Override public byte[] getTextBytes(int fieldId) { - return getText(fieldId).getBytes(TextDatum.DEFAULT_CHARSET); + return asDatum(fieldId).asTextBytes(); } @Override @@ -232,7 +232,7 @@ public double getFloat8(int fieldId) { @Override public String getText(int fieldId) { - return asDatum(fieldId).asChars(); + return new String(getBytes(fieldId), TextDatum.DEFAULT_CHARSET); } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java index d909440e41..e8f219cebc 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/OffHeapRowBlockUtils.java @@ -117,7 +117,7 @@ public static void convert(Tuple tuple, RowWriter writer) { break; case CHAR: case TEXT: - writer.putText(tuple.getTextBytes(i)); + writer.putText(tuple.getBytes(i)); break; case BLOB: writer.putBlob(tuple.getBytes(i)); diff --git a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java index ef6a9bd7eb..3dc8c23341 100644 --- a/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/tuple/memory/UnSafeTuple.java @@ -239,7 +239,7 @@ public byte[] getBytes(int fieldId) { @Override public byte[] getTextBytes(int fieldId) { - return getText(fieldId).getBytes(TextDatum.DEFAULT_CHARSET); + return asDatum(fieldId).asTextBytes(); } @Override @@ -270,9 +270,10 @@ public double getFloat8(int fieldId) { @Override public String getText(int fieldId) { - return asDatum(fieldId).asChars(); + return new String(getBytes(fieldId), TextDatum.DEFAULT_CHARSET); } + @Override public IntervalDatum getInterval(int fieldId) { long pos = getFieldAddr(fieldId); int months = PlatformDependent.getInt(pos); @@ -309,7 +310,7 @@ public char[] getUnicodeChars(int fieldId) { @Override public TimeMeta getTimeDate(int fieldId) { - return null; + return asDatum(fieldId).asTimeMeta(); } @Override