From cc7a710a1ba8d050836f64d820f675546712b3c9 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Aug 2018 02:59:18 +0000 Subject: [PATCH 1/2] Reduce the size of acquired arrays to avoid OOM error. --- .../apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java index 03cec8ed81b72..929c052ae63b1 100644 --- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java +++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java @@ -387,8 +387,8 @@ public void randomizedStressTest() { try { // Fill the map to 90% full so that we can trigger probing for (int i = 0; i < size * 0.9; i++) { - final byte[] key = getRandomByteArray(rand.nextInt(256) + 1); - final byte[] value = getRandomByteArray(rand.nextInt(512) + 1); + final byte[] key = getRandomByteArray(rand.nextInt(128) + 1); + final byte[] value = getRandomByteArray(rand.nextInt(128) + 1); if (!expected.containsKey(ByteBuffer.wrap(key))) { expected.put(ByteBuffer.wrap(key), value); final BytesToBytesMap.Location loc = map.lookup( From 737d9b151e18b521e2f57ac4c668a9512dba9389 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Aug 2018 07:39:01 +0000 Subject: [PATCH 2/2] Tweak map and array size. --- .../spark/unsafe/map/AbstractBytesToBytesMapSuite.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java index 929c052ae63b1..53a233f698c7a 100644 --- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java +++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java @@ -379,7 +379,7 @@ public void iteratingOverDataPagesWithWastedSpace() throws Exception { @Test public void randomizedStressTest() { - final int size = 65536; + final int size = 32768; // Java arrays' hashCodes() aren't based on the arrays' contents, so we need to wrap arrays // into ByteBuffers in order to use them as keys here. final Map expected = new HashMap<>(); @@ -387,8 +387,8 @@ public void randomizedStressTest() { try { // Fill the map to 90% full so that we can trigger probing for (int i = 0; i < size * 0.9; i++) { - final byte[] key = getRandomByteArray(rand.nextInt(128) + 1); - final byte[] value = getRandomByteArray(rand.nextInt(128) + 1); + final byte[] key = getRandomByteArray(rand.nextInt(256) + 1); + final byte[] value = getRandomByteArray(rand.nextInt(256) + 1); if (!expected.containsKey(ByteBuffer.wrap(key))) { expected.put(ByteBuffer.wrap(key), value); final BytesToBytesMap.Location loc = map.lookup(