From c62c1da808dd8c14bea3c8a391923c7f38c17bd1 Mon Sep 17 00:00:00 2001 From: Romain Yon Date: Fri, 19 May 2017 16:16:24 -0400 Subject: [PATCH 1/2] [BEAM-2334] Fix OutOfMemoryError --- .../org/apache/beam/runners/dataflow/util/RandomAccessData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java index 5ea9f07ff6a9..1deaaa210b86 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java @@ -350,7 +350,7 @@ private void ensureCapacity(int minCapacity) { // Try to double the size of the buffer, if thats not enough, just use the new capacity. // Note that we use Math.min(long, long) to not cause overflow on the multiplication. - int newCapacity = (int) Math.min(Integer.MAX_VALUE, buffer.length * 2L); + int newCapacity = (int) Math.min(Integer.MAX_VALUE - 2, buffer.length * 2L); if (newCapacity < minCapacity) { newCapacity = minCapacity; } From a4b2bea5e5600ee384c99fc0fbf9e72518962652 Mon Sep 17 00:00:00 2001 From: Romain Yon Date: Fri, 19 May 2017 19:29:20 -0400 Subject: [PATCH 2/2] set safer max Array size based on http://stackoverflow.com/a/8381338 --- .../org/apache/beam/runners/dataflow/util/RandomAccessData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java index 1deaaa210b86..0c0890243d25 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java @@ -350,7 +350,7 @@ private void ensureCapacity(int minCapacity) { // Try to double the size of the buffer, if thats not enough, just use the new capacity. // Note that we use Math.min(long, long) to not cause overflow on the multiplication. - int newCapacity = (int) Math.min(Integer.MAX_VALUE - 2, buffer.length * 2L); + int newCapacity = (int) Math.min(Integer.MAX_VALUE - 8, buffer.length * 2L); if (newCapacity < minCapacity) { newCapacity = minCapacity; }