Skip to content
Permalink
Browse files
Interim commit
  • Loading branch information
leerho committed Aug 15, 2021
1 parent e3a0c37 commit b4b17f1fda3ba78515bc8dcbdfd4aea8c00d2772
Show file tree
Hide file tree
Showing 55 changed files with 1,000 additions and 1,792 deletions.
@@ -21,10 +21,10 @@

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.fail;

import org.apache.datasketches.memory.BaseState;
import org.apache.datasketches.memory.DefaultMemoryRequestServer;
import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
@@ -72,8 +72,13 @@ public void checkDefaultMemoryRequestServer() throws Exception {

int longs2 = 64;
int bytes2 = longs2 << 3;
MemoryRequestServer memReqSvr = origWmem.getMemoryRequestServer();
WritableMemory newWmem = memReqSvr.request(bytes2);
MemoryRequestServer memReqSvr;
if (BaseState.defaultMemReqSvr == null) {
memReqSvr = new DefaultMemoryRequestServer();
} else {
memReqSvr = origWmem.getMemoryRequestServer();
}
WritableMemory newWmem = memReqSvr.request(origWmem, bytes2);
assertFalse(newWmem.isDirect()); //on heap by default
for (int i = 0; i < longs2; i++) {
newWmem.putLong(i << 3, i);
@@ -85,18 +90,9 @@ public void checkDefaultMemoryRequestServer() throws Exception {
}
}

@Test
public void checkNullMemoryRequestServer() throws Exception {
try (WritableHandle wh = WritableMemory.allocateDirect(128, Util.nativeByteOrder, null)) {
WritableMemory wmem = wh.getWritable();
assertNotNull(wmem.getMemoryRequestServer());
}
}


@Test
public void checkNonNativeDirect() throws Exception {
try (WritableHandle h = WritableMemory.allocateDirect(128, Util.nonNativeByteOrder, null)) {
try (WritableHandle h = WritableMemory.allocateDirect(128, Util.NON_NATIVE_BYTE_ORDER, null)) {
WritableMemory wmem = h.getWritable();
wmem.putChar(0, (char) 1);
assertEquals(wmem.getByte(1), (byte) 1);
@@ -41,10 +41,10 @@
import org.apache.datasketches.memory.BaseState;
import org.apache.datasketches.memory.MapHandle;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.ReadOnlyException;
import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMapHandle;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.memory.internal.ReadOnlyException;
import org.apache.datasketches.memory.internal.Util;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -147,7 +147,7 @@ public void checkNonNativeFile() throws Exception {
file.deleteOnExit(); //comment out if you want to examine the file.

final long bytes = 8;
try (WritableMapHandle h = WritableMemory.writableMap(file, 0L, bytes, Util.nonNativeByteOrder)) {
try (WritableMapHandle h = WritableMemory.writableMap(file, 0L, bytes, Util.NON_NATIVE_BYTE_ORDER)) {
WritableMemory wmem = h.getWritable();
wmem.putChar(0, (char) 1);
assertEquals(wmem.getByte(1), (byte) 1);
@@ -166,15 +166,17 @@ public void testMapException() throws IOException {
public void simpleMap2() throws Exception {
File file = getResourceFile("GettysburgAddress.txt");
assertTrue(isFileReadOnly(file));
try (WritableMapHandle rh = WritableMemory.writableMap(file)) { //throws
try (WritableMapHandle rh =
WritableMemory.writableMap(file)) { //throws
//
}
}

@Test(expectedExceptions = IllegalArgumentException.class)
@Test(expectedExceptions = ReadOnlyException.class)
public void checkOverLength() throws Exception {
File file = getResourceFile("GettysburgAddress.txt");
try (WritableMapHandle rh = WritableMemory.writableMap(file, 0, 1 << 20, ByteOrder.nativeOrder())) {
try (WritableMapHandle rh =
WritableMemory.writableMap(file, 0, 1 << 20, ByteOrder.nativeOrder())) {
//
} catch (IOException e) {
throw new RuntimeException(e);
@@ -21,9 +21,9 @@

import static org.testng.Assert.fail;

import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.Buffer;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;

@@ -82,7 +82,9 @@ public void checkCheckValid() throws Exception {
wmem = hand.getWritable();
buf = wmem.asBuffer();
}
@SuppressWarnings("unused")
Memory mem = buf.asMemory();
try {
@SuppressWarnings("unused")
Memory mem = buf.asMemory();
} catch (AssertionError ae) { }
}
}
@@ -30,9 +30,9 @@
import org.apache.datasketches.memory.Buffer;
import org.apache.datasketches.memory.DefaultMemoryRequestServer;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.ReadOnlyException;
import org.apache.datasketches.memory.WritableBuffer;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.memory.internal.ReadOnlyException;
import org.testng.annotations.Test;

@SuppressWarnings("javadoc")
@@ -22,7 +22,7 @@
import java.nio.ByteBuffer;

import org.apache.datasketches.memory.Buffer;
import org.apache.datasketches.memory.internal.ReadOnlyException;
import org.apache.datasketches.memory.ReadOnlyException;
import org.apache.datasketches.memory.WritableBuffer;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;
@@ -27,6 +27,7 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import org.apache.datasketches.memory.DefaultMemoryRequestServer;
import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;
@@ -65,9 +66,10 @@ public void withByteBuffer() {

//Request Bigger Memory
MemoryRequestServer svr = mem.getMemoryRequestServer();
if (svr == null) { svr = new DefaultMemoryRequestServer(); }
assertNotNull(svr); //before the fix, this was null.

WritableMemory newMem = svr.request(initialMemSize * 2);
WritableMemory newMem = svr.request(mem, initialMemSize * 2);

//Confirm that newMem is on the heap (the default) and 2X size
assertFalse(newMem.isDirect());
@@ -37,21 +37,23 @@
public class ExampleMemoryRequestServerTest {

/**
* In this version all of the memory allocations are done through the MemoryRequestServer
* This version is without a TWR block.all of the memory allocations are done through the MemoryRequestServer
* and each is closed by the MemoryClient when it is done with each.
*/
@SuppressWarnings("resource")
@Test
public void checkExampleMemoryRequestServer1() {
int bytes = 8;
ExampleMemoryRequestServer svr = new ExampleMemoryRequestServer();
WritableMemory wMem = svr.request(bytes);
WritableMemory memStart = WritableMemory.allocateDirect(8).getWritable();
WritableMemory wMem = svr.request(memStart, bytes);
MemoryClient client = new MemoryClient(wMem);
client.process();
svr.cleanup();
}

/**
* In this version the first memory allocation is done separately.
* In this version the first memory allocation is done up front in a TWR block.
* And then the MemoryClient allocates new memories as needed, which are then closed
* by the MemoryClient when it is done with the new memory allocations.
* The initial allocation stays open until the end where it is closed at the end of the
@@ -63,10 +65,10 @@ public void checkExampleMemoryRequestServer2() throws Exception {
int bytes = 8;
ExampleMemoryRequestServer svr = new ExampleMemoryRequestServer();
try (WritableHandle handle = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), svr)) {
WritableMemory wMem = handle.getWritable();
MemoryClient client = new MemoryClient(wMem);
WritableMemory memStart = handle.getWritable();
MemoryClient client = new MemoryClient(memStart);
client.process();
svr.cleanup();
svr.cleanup(); //just to be sure all are closed.
}
}

@@ -88,25 +90,25 @@ static class MemoryClient {
WritableMemory smallMem;
MemoryRequestServer svr;

MemoryClient(WritableMemory wmem) {
smallMem = wmem;
svr = wmem.getMemoryRequestServer();
MemoryClient(WritableMemory memStart) {
smallMem = memStart;
svr = memStart.getMemoryRequestServer();
}

void process() {
long cap1 = smallMem.getCapacity();
smallMem.fill((byte) 1); //fill it, but not big enough
println(smallMem.toHexString("Small", 0, (int)cap1));

WritableMemory bigMem = svr.request(2 * cap1); //get bigger mem
WritableMemory bigMem = svr.request(smallMem, 2 * cap1); //get bigger mem
long cap2 = bigMem.getCapacity();
smallMem.copyTo(0, bigMem, 0, cap1); //copy data from small to big
svr.requestClose(smallMem, bigMem); //done with smallMem, release it

bigMem.fill(cap1, cap1, (byte) 2); //fill the rest of bigMem, still not big enough
println(bigMem.toHexString("Big", 0, (int)cap2));

WritableMemory giantMem = svr.request(2 * cap2); //get giant mem
WritableMemory giantMem = svr.request(bigMem, 2 * cap2); //get giant mem
long cap3 = giantMem.getCapacity();
bigMem.copyTo(0, giantMem, 0, cap2); //copy data from small to big
svr.requestClose(bigMem, giantMem); //done with bigMem, release it
@@ -119,15 +121,17 @@ void process() {

/**
* This example MemoryRequestServer is simplistic but demonstrates one of many ways to
* possibly manage the continuous requests for larger memory.
* possibly manage the continuous requests for larger memory and to track the associations between
* handles and their associated memory.
*/
public static class ExampleMemoryRequestServer implements MemoryRequestServer {
IdentityHashMap<WritableMemory, WritableHandle> map = new IdentityHashMap<>();

@SuppressWarnings("resource")
@Override
public WritableMemory request(long capacityBytes) {
WritableHandle handle = WritableMemory.allocateDirect(capacityBytes, ByteOrder.nativeOrder(), this);
public WritableMemory request(WritableMemory currentWMem, long capacityBytes) {
ByteOrder order = currentWMem.getTypeByteOrder();
WritableHandle handle = WritableMemory.allocateDirect(capacityBytes, order, this);
WritableMemory wmem = handle.getWritable();
map.put(wmem, handle); //We track the newly allocated memory and its handle.
return wmem;
@@ -137,14 +141,12 @@ public WritableMemory request(long capacityBytes) {
@Override
//here we actually release it, in reality it might be a lot more complex.
public void requestClose(WritableMemory memToRelease, WritableMemory newMemory) {
if (memToRelease != null) {
WritableHandle handle = map.get(memToRelease);
if (handle != null && handle.getWritable() == memToRelease) {
try {
handle.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
WritableHandle handle = map.get(memToRelease);
if (handle != null && handle.getWritable() == memToRelease) {
try {
handle.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
@@ -156,7 +158,7 @@ public void cleanup() {
v.close();
} catch (Exception e) {
throw new RuntimeException(e);
} //harmless
}
});
}
}

0 comments on commit b4b17f1

Please sign in to comment.