From feaeba923192a7a22db37e3cd1353ad60f4ef4f0 Mon Sep 17 00:00:00 2001 From: roryqi Date: Tue, 7 Mar 2023 20:57:57 +0800 Subject: [PATCH 1/3] [#585] feat(netty): Add MaxDirectMemorySize option for shuffle Server --- bin/rss-env.sh | 1 + bin/start-shuffle-server.sh | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/bin/rss-env.sh b/bin/rss-env.sh index 14824273a1..a254dad21c 100644 --- a/bin/rss-env.sh +++ b/bin/rss-env.sh @@ -31,3 +31,4 @@ XMX_SIZE="80g" # Shuffle Server JVM XMX size # RSS_PID_DIR, Where the pid file is stored (Default: ${RSS_HOME}) # RSS_LOG_DIR, Where log files are stored (Default: ${RSS_HOME}/logs) # RSS_IP, IP address Shuffle Server binds to on this node (Default: first non-loopback ipv4) +# MAX_DIRECT_MEMORY_SIZE Shuffle Server JVM off heap memory size (Default: 0) diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh index e0b979e9a8..4bfb340006 100644 --- a/bin/start-shuffle-server.sh +++ b/bin/start-shuffle-server.sh @@ -66,9 +66,15 @@ JAVA_LIB_PATH="-Djava.library.path=$HADOOP_HOME/lib/native" echo "class path is $CLASSPATH" +MAX_DIRECT_MEMORY_OPTS = "" +if [ -n "$MAX_DIRECT_MEMORY_SIZE" ]; then + MAX_DIRECT_MEMORY_OPTS = "-XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY_SIZE" +fi + JVM_ARGS=" -server \ -Xmx${XMX_SIZE} \ -Xms${XMX_SIZE} \ + ${MAX_DIRECT_MEMORY_OPTS} \ -XX:+UseG1GC \ -XX:MaxGCPauseMillis=200 \ -XX:ParallelGCThreads=20 \ From 17b770ee194057f8b6ba7aee5b07dd663416d6c9 Mon Sep 17 00:00:00 2001 From: roryqi Date: Tue, 7 Mar 2023 21:08:15 +0800 Subject: [PATCH 2/3] fix --- bin/start-shuffle-server.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh index 4bfb340006..829a20db2f 100644 --- a/bin/start-shuffle-server.sh +++ b/bin/start-shuffle-server.sh @@ -66,9 +66,9 @@ JAVA_LIB_PATH="-Djava.library.path=$HADOOP_HOME/lib/native" echo "class path is $CLASSPATH" -MAX_DIRECT_MEMORY_OPTS = "" +MAX_DIRECT_MEMORY_OPTS="" if [ -n "$MAX_DIRECT_MEMORY_SIZE" ]; then - MAX_DIRECT_MEMORY_OPTS = "-XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY_SIZE" + MAX_DIRECT_MEMORY_OPTS="-XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY_SIZE" fi JVM_ARGS=" -server \ From 4ca9e1327f308e6f27fe178c07f0b1b329d9f6b7 Mon Sep 17 00:00:00 2001 From: roryqi Date: Tue, 7 Mar 2023 21:09:24 +0800 Subject: [PATCH 3/3] fix --- bin/rss-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/rss-env.sh b/bin/rss-env.sh index a254dad21c..b46ec3af16 100644 --- a/bin/rss-env.sh +++ b/bin/rss-env.sh @@ -31,4 +31,4 @@ XMX_SIZE="80g" # Shuffle Server JVM XMX size # RSS_PID_DIR, Where the pid file is stored (Default: ${RSS_HOME}) # RSS_LOG_DIR, Where log files are stored (Default: ${RSS_HOME}/logs) # RSS_IP, IP address Shuffle Server binds to on this node (Default: first non-loopback ipv4) -# MAX_DIRECT_MEMORY_SIZE Shuffle Server JVM off heap memory size (Default: 0) +# MAX_DIRECT_MEMORY_SIZE Shuffle Server JVM off heap memory size (Default: not set)