From 38ca07b1f65d4d16d8ccf0a5cfbc570c2c7882e1 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Mon, 5 Sep 2016 14:08:05 +0100 Subject: [PATCH 1/4] Upgrade netty-all to 4.0.41 final for bug fixes --- dev/deps/spark-deps-hadoop-2.2 | 2 +- dev/deps/spark-deps-hadoop-2.3 | 2 +- dev/deps/spark-deps-hadoop-2.4 | 2 +- dev/deps/spark-deps-hadoop-2.6 | 2 +- dev/deps/spark-deps-hadoop-2.7 | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index eaed0889ac36f..44e75d1523844 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -124,7 +124,7 @@ metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.3.0.jar netty-3.8.0.Final.jar -netty-all-4.0.29.Final.jar +netty-all-4.0.41.Final.jar objenesis-2.1.jar opencsv-2.3.jar oro-2.0.8.jar diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index d68a7f462ba7f..e2fc21ac5530a 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -131,7 +131,7 @@ metrics-jvm-3.1.2.jar minlog-1.3.0.jar mx4j-3.0.2.jar netty-3.8.0.Final.jar -netty-all-4.0.29.Final.jar +netty-all-4.0.41.Final.jar objenesis-2.1.jar opencsv-2.3.jar oro-2.0.8.jar diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 346f19767d367..6701cac7c980f 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -131,7 +131,7 @@ metrics-jvm-3.1.2.jar minlog-1.3.0.jar mx4j-3.0.2.jar netty-3.8.0.Final.jar -netty-all-4.0.29.Final.jar +netty-all-4.0.41.Final.jar objenesis-2.1.jar opencsv-2.3.jar oro-2.0.8.jar diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 6f4695f345a48..12ee1e47b9633 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -139,7 +139,7 @@ metrics-jvm-3.1.2.jar minlog-1.3.0.jar mx4j-3.0.2.jar netty-3.8.0.Final.jar -netty-all-4.0.29.Final.jar +netty-all-4.0.41.Final.jar objenesis-2.1.jar opencsv-2.3.jar oro-2.0.8.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 7a86a8bd88846..c1e54fbb317b2 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -140,7 +140,7 @@ metrics-jvm-3.1.2.jar minlog-1.3.0.jar mx4j-3.0.2.jar netty-3.8.0.Final.jar -netty-all-4.0.29.Final.jar +netty-all-4.0.41.Final.jar objenesis-2.1.jar opencsv-2.3.jar oro-2.0.8.jar diff --git a/pom.xml b/pom.xml index 2c265c1fa325e..990fbd777bc43 100644 --- a/pom.xml +++ b/pom.xml @@ -539,7 +539,7 @@ io.netty netty-all - 4.0.29.Final + 4.0.41.Final io.netty From 502ebf45f4fa9791cbf26ec5ea7e0167ecbc68a0 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Mon, 12 Sep 2016 16:25:37 +0100 Subject: [PATCH 2/4] Set io.netty.recycler.maxCapacity in TransportConf Based on suggestions by zsxwing and Sean Owen as we see failures when upgrading to 4.0.41 final --- .../java/org/apache/spark/network/util/TransportConf.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 0efc400aa388c..b6f878ba58937 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -23,6 +23,10 @@ * A central location that tracks all the settings we expose to users. */ public class TransportConf { + + static { + System.setProperty("io.netty.recycler.maxCapacity", "0"); + } private final String SPARK_NETWORK_IO_MODE_KEY; private final String SPARK_NETWORK_IO_PREFERDIRECTBUFS_KEY; From faefd9ca2975830b6afff0ed78ac5e90c69ec9f4 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Tue, 13 Sep 2016 15:10:33 +0100 Subject: [PATCH 3/4] Use unpooled byte buffer Noticing an exception with pooled byte buffer when upgrading to netty 4.0.41.final so let's see if this helps --- .../main/java/org/apache/spark/network/util/TransportConf.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index b6f878ba58937..ef9dfcba77a56 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -25,7 +25,7 @@ public class TransportConf { static { - System.setProperty("io.netty.recycler.maxCapacity", "0"); + System.setProperty("io.netty.allocator.type", "unpooled"); } private final String SPARK_NETWORK_IO_MODE_KEY; From 8f6783b37c6e7613750281adee03c6c343e4415a Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 14 Sep 2016 09:36:57 +0100 Subject: [PATCH 4/4] Set maxDirectMemory to 0 for netty Thanks to zsxwing who noticed an issue when upgrading netty to 4.0.37+ which is caused by netty/netty@d58dec8 --- .../main/java/org/apache/spark/network/util/TransportConf.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index ef9dfcba77a56..7d5baa9a9c8f8 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -25,7 +25,8 @@ public class TransportConf { static { - System.setProperty("io.netty.allocator.type", "unpooled"); + // Set this due to Netty PR #5661 for Netty 4.0.37+ to work + System.setProperty("io.netty.maxDirectMemory", "0"); } private final String SPARK_NETWORK_IO_MODE_KEY;