Skip to content
Permalink
Browse files
Merge pull request #138 from apache/addMimicDruidTest
This test mimics the Druid issue #11544, but at a much smaller scale.
  • Loading branch information
leerho committed Aug 10, 2021
2 parents e97d072 + edc2003 commit 8cb5f3945ee630e1c3b1ac6d887a5505afc86770
Show file tree
Hide file tree
Showing 6 changed files with 572 additions and 7 deletions.
@@ -29,16 +29,17 @@

import org.apache.datasketches.memory.Buffer;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableBuffer;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.memory.internal.BaseStateImpl;
import org.apache.datasketches.memory.internal.Prim;
import org.apache.datasketches.memory.internal.StepBoolean;
import org.apache.datasketches.memory.internal.Util;
import org.apache.datasketches.memory.WritableBuffer;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;

@SuppressWarnings("javadoc")
public class BaseStateTest {

@Test
public void checkPrimOffset() {
int off = (int)Prim.BYTE.off();
@@ -107,6 +108,30 @@ public void checkByteOrderNull() {
fail();
}

@Test
public void checkIsNativeByteOrder() {
assertTrue(BaseStateImpl.isNativeByteOrder(ByteOrder.nativeOrder()));
try {
BaseStateImpl.isNativeByteOrder(null);
fail();
} catch (final IllegalArgumentException e) {}
}

@Test
public void checkXxHash64() {
WritableMemory mem = WritableMemory.allocate(8);
long out = mem.xxHash64(mem.getLong(0), 1L);
assertTrue(out != 0);
}

@Test
public void checkTypeDecode() {
for (int i = 0; i < 128; i++) {
BaseStateImpl.typeDecode(i);
}
}

/********************/
@Test
public void printlnTest() {
println("PRINTING: "+this.getClass().getName());
@@ -21,16 +21,18 @@

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import org.apache.datasketches.memory.Buffer;
import org.apache.datasketches.memory.DefaultMemoryRequestServer;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.internal.ReadOnlyException;
import org.apache.datasketches.memory.WritableBuffer;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.memory.internal.ReadOnlyException;
import org.testng.annotations.Test;

@SuppressWarnings("javadoc")
@@ -406,6 +408,15 @@ public void testIllegalFill() {
wbuf.fill((byte)0);
}

@Test
public void checkWritableWrap() {
ByteBuffer bb = ByteBuffer.allocate(16);
WritableBuffer buf = WritableBuffer.writableWrap(bb, ByteOrder.nativeOrder(), null);
assertNotNull(buf);
buf = WritableBuffer.writableWrap(bb, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
assertNotNull(buf);
}

@Test
public void testWritableDuplicate() {
WritableMemory wmem = WritableMemory.writableWrap(new byte[1]);
@@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.datasketches.memory.test;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;

/**
* The original design provided the MemoryRequestServer callback only for Memory segments allocated via
* <i>WritableMemory.allocateDirect(...)</i> calls. Memory segments allocated via
* <i>WritableMemory.wrap(ByteBuffer)</i> did not have this capability. This was a major oversight since
* all off-heap memory in Druid is allocated using ByteBuffers! It is unusual that no one has
* uncovered this until August 2021. Nonetheless, the fix involves instrumenting all the paths involved
* in providing this callback mechanism for wrapped ByteBuffers.
*
* This issues was first identified in Druid Issue #11544 and then posted as DataSketches-java Issue #358.
* But the actual source of the problem was in Memory.
*
* This test mimics the Druid issue but at a much smaller scale.
*
* @author Lee Rhodes
*
*/
public class DruidIssue11544Test {

@Test
public void withByteBuffer() {
int initialLongs = 1000;
int initialMemSize = initialLongs * 8;
ByteBuffer bb = ByteBuffer.allocateDirect(initialMemSize);
bb.order(ByteOrder.nativeOrder());

//Fill the byte buffer
for (int i = 0; i < initialLongs; i++) { bb.putLong(i * 8, i); }

//Wrap, assuming default MemoryRequestServer
WritableMemory mem = WritableMemory.writableWrap(bb);
assertTrue(mem.isDirect()); //confirm mem is off-heap

//Request Bigger Memory
MemoryRequestServer svr = mem.getMemoryRequestServer();
assertNotNull(svr); //before the fix, this was null.

WritableMemory newMem = svr.request(initialMemSize * 2);

//Confirm that newMem is on the heap (the default) and 2X size
assertFalse(newMem.isDirect());
assertEquals(newMem.getCapacity(), 2 * initialMemSize);

//Move data to new memory
mem.copyTo(0, newMem, 0, initialMemSize);

//Prepare to request deallocation
WritableMemory oldMem = mem;
mem = newMem;

//In the DefaultMemoryRequestServer, this is a no-op, so nothing is actually deallocated.
svr.requestClose(oldMem, newMem);
assertTrue(oldMem.isValid());
assertTrue(mem.isValid());
}

}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.datasketches.memory.test;

import org.apache.datasketches.memory.MemoryCloseException;
import org.testng.annotations.Test;

public class MemoryCloseExceptionTest {

@Test
public void checkNoArgs() {
try {
throw new MemoryCloseException();
} catch (final MemoryCloseException e) {}

try {
throw new MemoryCloseException("Test Exception");
} catch (final MemoryCloseException e) {}
}
}


0 comments on commit 8cb5f39

Please sign in to comment.