Skip to content
Permalink
Browse files
More clean up
  • Loading branch information
leerho committed Aug 16, 2021
1 parent dfa2e92 commit 543b461fd4ac6434f7d677e58719b6e1eefa1d28
Show file tree
Hide file tree
Showing 8 changed files with 66 additions and 68 deletions.
@@ -53,39 +53,58 @@ public class DruidIssue11544Test {
@Test
public void withByteBuffer() {
int initialLongs = 1000;
int initialMemSize = initialLongs * 8;
ByteBuffer bb = ByteBuffer.allocateDirect(initialMemSize);
int size1 = initialLongs * 8;

//Start with a ByteBuffer
ByteBuffer bb = ByteBuffer.allocateDirect(size1);
bb.order(ByteOrder.nativeOrder());

//Fill the byte buffer
for (int i = 0; i < initialLongs; i++) { bb.putLong(i * 8, i); }
//Wrap bb into WritableMemory
WritableMemory mem1 = WritableMemory.writableWrap(bb);
assertTrue(mem1.isDirect()); //confirm mem1 is off-heap

//Wrap, assuming default MemoryRequestServer
WritableMemory mem = WritableMemory.writableWrap(bb);
assertTrue(mem.isDirect()); //confirm mem is off-heap
//Acquire the DefaultMemoryRequestServer
//NOTE: it is a policy decision to allow the DefaultMemoryServer to be set as a default.
// It might be set to null. So we need to check what the current policy is.
MemoryRequestServer svr = mem1.getMemoryRequestServer();
if (svr == null) {
svr = new DefaultMemoryRequestServer();
}
assertNotNull(svr);

//Request Bigger Memory
MemoryRequestServer svr = mem.getMemoryRequestServer();
if (svr == null) { svr = new DefaultMemoryRequestServer(); }
assertNotNull(svr); //before the fix, this was null.
int size2 = size1 * 2;
WritableMemory mem2 = svr.request(mem1, size2);

//Confirm that mem2 is on the heap (the default) and 2X size1
assertFalse(mem2.isDirect());
assertEquals(mem2.getCapacity(), size2);

//Move data to new memory
mem1.copyTo(0, mem2, 0, size1);

WritableMemory newMem = svr.request(mem, initialMemSize * 2);
//Prepare to request deallocation
//In the DefaultMemoryRequestServer, this is a no-op, so nothing is actually deallocated.
svr.requestClose(mem1, mem2);
assertTrue(mem1.isValid());
assertTrue(mem2.isValid());

//Confirm that newMem is on the heap (the default) and 2X size
assertFalse(newMem.isDirect());
assertEquals(newMem.getCapacity(), 2 * initialMemSize);
//Now we are on the heap and need to grow again:
int size3 = size2 * 2;
WritableMemory mem3 = svr.request(mem2, size3);

//Confirm that mem3 is still on the heap and 2X of size2
assertFalse(mem3.isDirect());
assertEquals(mem3.getCapacity(), size3);

//Move data to new memory
mem.copyTo(0, newMem, 0, initialMemSize);
mem2.copyTo(0, mem3, 0, size2);

//Prepare to request deallocation
WritableMemory oldMem = mem;
mem = newMem;

//In the DefaultMemoryRequestServer, this is a no-op, so nothing is actually deallocated.
svr.requestClose(oldMem, newMem);
assertTrue(oldMem.isValid());
assertTrue(mem.isValid());
svr.requestClose(mem2, mem3); //No-op
assertTrue(mem2.isValid());
assertTrue(mem3.isValid());
}

}
@@ -19,7 +19,6 @@

package org.apache.datasketches.memory.test;

import static org.apache.datasketches.memory.internal.Util.NATIVE_BYTE_ORDER;
import static org.apache.datasketches.memory.internal.Util.NON_NATIVE_BYTE_ORDER;
import static org.apache.datasketches.memory.internal.Util.otherByteOrder;
import static org.testng.Assert.assertEquals;
@@ -45,7 +44,7 @@
*/
@SuppressWarnings("javadoc")
public class LeafImplTest {
private static final ByteOrder NBO = NATIVE_BYTE_ORDER;
private static final ByteOrder NBO = ByteOrder.nativeOrder();
private static final ByteOrder NNBO = NON_NATIVE_BYTE_ORDER;
private static final MemoryRequestServer dummyMemReqSvr = new DummyMemoryRequestServer();

@@ -212,7 +212,7 @@ public void checkByteBufWrongOrder() {
ByteBuffer bb = ByteBuffer.allocate(n * 8);
bb.order(ByteOrder.BIG_ENDIAN);
Memory mem = Memory.wrap(bb);
assertFalse(mem.getTypeByteOrder() == Util.NATIVE_BYTE_ORDER);
assertFalse(mem.getTypeByteOrder() == ByteOrder.nativeOrder());
assertEquals(mem.getTypeByteOrder(), ByteOrder.BIG_ENDIAN);
}

@@ -25,6 +25,7 @@
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import org.apache.datasketches.memory.Buffer;
import org.apache.datasketches.memory.Memory;
@@ -44,13 +45,13 @@ public class SpecificLeafTest {
public void checkByteBufferLeafs() {
int bytes = 128;
ByteBuffer bb = ByteBuffer.allocate(bytes);
bb.order(Util.NATIVE_BYTE_ORDER);
bb.order(ByteOrder.nativeOrder());

Memory mem = Memory.wrap(bb).region(0, bytes, Util.NATIVE_BYTE_ORDER);
Memory mem = Memory.wrap(bb).region(0, bytes, ByteOrder.nativeOrder());
assertTrue(ReflectUtil.isBBType(mem));
assertTrue(mem.isReadOnly());
checkCrossLeafTypeIds(mem);
Buffer buf = mem.asBuffer().region(0, bytes, Util.NATIVE_BYTE_ORDER);
Buffer buf = mem.asBuffer().region(0, bytes, ByteOrder.nativeOrder());

bb.order(Util.NON_NATIVE_BYTE_ORDER);
Memory mem2 = Memory.wrap(bb).region(0, bytes, Util.NON_NATIVE_BYTE_ORDER);
@@ -74,8 +75,8 @@ public void checkDirectLeafs() throws Exception {
checkCrossLeafTypeIds(wmem);
WritableMemory nnwmem = wmem.writableRegion(0, bytes, Util.NON_NATIVE_BYTE_ORDER);

Memory mem = wmem.region(0, bytes, Util.NATIVE_BYTE_ORDER);
Buffer buf = mem.asBuffer().region(0, bytes, Util.NATIVE_BYTE_ORDER);
Memory mem = wmem.region(0, bytes, ByteOrder.nativeOrder());
Buffer buf = mem.asBuffer().region(0, bytes, ByteOrder.nativeOrder());


Memory mem2 = nnwmem.region(0, bytes, Util.NON_NATIVE_BYTE_ORDER);
@@ -107,15 +108,15 @@ public void checkMapLeafs() throws Exception {

final long bytes = 128;

try (WritableMapHandle h = WritableMemory.writableMap(file, 0L, bytes, Util.NATIVE_BYTE_ORDER)) {
try (WritableMapHandle h = WritableMemory.writableMap(file, 0L, bytes, ByteOrder.nativeOrder())) {
WritableMemory mem = h.getWritable(); //native mem
assertTrue(ReflectUtil.isMapType(mem));
assertFalse(mem.isReadOnly());
checkCrossLeafTypeIds(mem);
Memory nnreg = mem.region(0, bytes, Util.NON_NATIVE_BYTE_ORDER);

Memory reg = mem.region(0, bytes, Util.NATIVE_BYTE_ORDER);
Buffer buf = reg.asBuffer().region(0, bytes, Util.NATIVE_BYTE_ORDER);
Memory reg = mem.region(0, bytes, ByteOrder.nativeOrder());
Buffer buf = reg.asBuffer().region(0, bytes, ByteOrder.nativeOrder());
Buffer buf4 = buf.duplicate();

Memory reg2 = nnreg.region(0, bytes, Util.NON_NATIVE_BYTE_ORDER);
@@ -140,8 +141,8 @@ public void checkHeapLeafs() {
checkCrossLeafTypeIds(mem);
Memory nnreg = mem.region(0, bytes, Util.NON_NATIVE_BYTE_ORDER);

Memory reg = mem.region(0, bytes, Util.NATIVE_BYTE_ORDER);
Buffer buf = reg.asBuffer().region(0, bytes, Util.NATIVE_BYTE_ORDER);
Memory reg = mem.region(0, bytes, ByteOrder.nativeOrder());
Buffer buf = reg.asBuffer().region(0, bytes, ByteOrder.nativeOrder());
Buffer buf4 = buf.duplicate();

Memory reg2 = nnreg.region(0, bytes, Util.NON_NATIVE_BYTE_ORDER);
@@ -41,6 +41,8 @@ public void wrapBigEndian() {
ByteBuffer bb = ByteBuffer.allocate(64); //big endian
WritableMemory wmem = WritableMemory.writableWrap(bb);
assertEquals(wmem.getTypeByteOrder(), ByteOrder.BIG_ENDIAN);
wmem = WritableMemory.writableWrap(bb, ByteOrder.nativeOrder());
assertEquals(wmem.getTypeByteOrder(), ByteOrder.LITTLE_ENDIAN);
}

@Test
@@ -57,6 +59,8 @@ public void allocateWithByteOrder() {
assertEquals(wmem.getTypeByteOrder(), ByteOrder.BIG_ENDIAN);
wmem = WritableMemory.allocate(64, ByteOrder.LITTLE_ENDIAN);
assertEquals(wmem.getTypeByteOrder(), ByteOrder.LITTLE_ENDIAN);
wmem = WritableMemory.writableWrap(new byte[64], 32, 32, ByteOrder.BIG_ENDIAN);
assertEquals(wmem.getTypeByteOrder(), ByteOrder.BIG_ENDIAN);
}

@Test
@@ -161,11 +165,11 @@ public void checkLargeEquals() {
@Test
public void checkWrapWithBO() {
WritableMemory wmem = WritableMemory.writableWrap(new byte[0], ByteOrder.BIG_ENDIAN);
boolean nativeBO = wmem.getTypeByteOrder() == Util.NATIVE_BYTE_ORDER;
boolean nativeBO = wmem.getTypeByteOrder() == ByteOrder.nativeOrder();
assertFalse(nativeBO);
println("" + nativeBO);
wmem = WritableMemory.writableWrap(new byte[8], ByteOrder.BIG_ENDIAN);
nativeBO = wmem.getTypeByteOrder() == Util.NATIVE_BYTE_ORDER;
nativeBO = wmem.getTypeByteOrder() == ByteOrder.nativeOrder();
assertFalse(nativeBO);
println("" + nativeBO);
}
@@ -101,7 +101,7 @@ public abstract class BaseStateImpl implements BaseState {

@Override
public final ByteOrder getTypeByteOrder() {
return isNonNativeType() ? Util.NON_NATIVE_BYTE_ORDER : Util.NATIVE_BYTE_ORDER;
return isNonNativeType() ? Util.NON_NATIVE_BYTE_ORDER : ByteOrder.nativeOrder();
}

/**
@@ -113,13 +113,13 @@ public static boolean isNativeByteOrder(final ByteOrder byteOrder) {
if (byteOrder == null) {
throw new IllegalArgumentException("ByteOrder parameter cannot be null.");
}
return Util.NATIVE_BYTE_ORDER == byteOrder;
return ByteOrder.nativeOrder() == byteOrder;
}

@Override
public final boolean isByteOrderCompatible(final ByteOrder byteOrder) {
final ByteOrder typeBO = getTypeByteOrder();
return typeBO == Util.NATIVE_BYTE_ORDER && typeBO == byteOrder;
return typeBO == ByteOrder.nativeOrder() && typeBO == byteOrder;
}

@Override
@@ -308,18 +308,10 @@ final boolean isRegionType() {
return (getTypeId() & REGION) > 0;
}

final static byte setRegionType(byte type, boolean region) {
return (byte)((type & ~2) | (region ? REGION : 0));
}

final boolean isDuplicateType() {
return (getTypeId() & DUPLICATE) > 0;
}

final static byte setDuplicateType(byte type, boolean duplicate) {
return (byte)((type & ~4) | (duplicate ? DUPLICATE : 0));
}

//The following are set by the leaf nodes
final boolean isBufferType() {
return (getTypeId() & BUFFER) > 0;
@@ -439,7 +431,7 @@ static final String toHex(final BaseStateImpl state, final String preamble, fina
sb.append("Valid : ").append(state.isValid()).append(LS);
sb.append("Read Only : ").append(state.isReadOnly()).append(LS);
sb.append("Type Byte Order : ").append(state.getTypeByteOrder().toString()).append(LS);
sb.append("Native Byte Order : ").append(Util.NATIVE_BYTE_ORDER.toString()).append(LS);
sb.append("Native Byte Order : ").append(ByteOrder.nativeOrder().toString()).append(LS);
sb.append("JDK Runtime Version : ").append(UnsafeUtil.JDK).append(LS);
//Data detail
sb.append("Data, littleEndian : 0 1 2 3 4 5 6 7");
@@ -67,14 +67,6 @@ public abstract class BaseWritableBufferImpl extends BaseBufferImpl implements W
super(unsafeObj, nativeBaseOffset, regionOffset, capacityBytes);
}

public static BaseWritableBufferImpl wrapHeapArray(final Object arr, final long offsetBytes, final long lengthBytes,
final boolean localReadOnly, final ByteOrder byteOrder, final MemoryRequestServer memReqSvr) {
final int typeId = localReadOnly ? READONLY : 0;
return Util.isNativeByteOrder(byteOrder)
? new HeapWritableBufferImpl(arr, offsetBytes, lengthBytes, typeId, memReqSvr)
: new HeapNonNativeWritableBufferImpl(arr, offsetBytes, lengthBytes, typeId, memReqSvr);
}

public static BaseWritableBufferImpl wrapByteBuffer(
final ByteBuffer byteBuf, final boolean localReadOnly, final ByteOrder byteOrder,
final MemoryRequestServer memReqSvr) {
@@ -38,12 +38,11 @@ public final class Util {
public static final String LS = System.getProperty("line.separator");

//Byte Order related
public static final ByteOrder NATIVE_BYTE_ORDER = ByteOrder.nativeOrder();
public static final ByteOrder NON_NATIVE_BYTE_ORDER = NATIVE_BYTE_ORDER == ByteOrder.LITTLE_ENDIAN
public static final ByteOrder NON_NATIVE_BYTE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN
? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;

public static ByteOrder otherByteOrder(ByteOrder order) {
return (order == NATIVE_BYTE_ORDER) ? NON_NATIVE_BYTE_ORDER : NATIVE_BYTE_ORDER;
return (order == ByteOrder.nativeOrder()) ? NON_NATIVE_BYTE_ORDER : ByteOrder.nativeOrder();
}


@@ -63,14 +62,6 @@ private Util() { }

//Byte Order Related

/**
* Returns the Native Byte Order
* @return the Native Byte Order
*/
public static final ByteOrder getNativeByteOrder() {
return NATIVE_BYTE_ORDER;
}

/**
* Returns true if the given byteOrder is the same as the native byte order.
* @param byteOrder the given byte order
@@ -80,7 +71,7 @@ public static boolean isNativeByteOrder(final ByteOrder byteOrder) {
if (byteOrder == null) {
throw new IllegalArgumentException("ByteOrder parameter cannot be null.");
}
return Util.NATIVE_BYTE_ORDER == byteOrder;
return ByteOrder.nativeOrder() == byteOrder;
}


0 comments on commit 543b461

Please sign in to comment.