Enable Netty and BookKeeper IO optimizations on jdk17 (#3234)

* Enable netty optimization and BK native IO on jdk17

* zip and better docs

bin/common.sh

@@ -284,7 +284,15 @@ build_cli_jvm_opts() {
 }
 
 build_netty_opts() {
-  echo "-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL}"
+  NETTY_OPTS="-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} -Dio.netty.tryReflectionSetAccessible=true"
+  # --add-opens does not exist on jdk8
+  if [ "$USING_JDK8" -eq "0" ]; then
+    # Enable java.nio.DirectByteBuffer
+    # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
+    # https://github.com/netty/netty/issues/12265
+    NETTY_OPTS="$NETTY_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+  fi
+  echo $NETTY_OPTS
 }
 
 build_logging_opts() {
@@ -316,7 +324,15 @@ build_cli_logging_opts() {
 }
 
 build_bookie_opts() {
-  echo "-Djava.net.preferIPv4Stack=true"
+  BOOKIE_OPTS="-Djava.net.preferIPv4Stack=true"
+  # --add-opens does not exist on jdk8
+  if [ "$USING_JDK8" -eq "0" ]; then
+    # enable posix_fadvise usage in the Journal
+    BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.io=ALL-UNNAMED"
+    # DirectMemoryCRC32Digest
+    BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
+  fi
+  echo $BOOKIE_OPTS
 }
 
 find_table_service() {

bin/common_gradle.sh

@@ -266,7 +266,15 @@ build_cli_jvm_opts() {
 }
 
 build_netty_opts() {
-  echo "-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL}"
+    NETTY_OPTS="-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} -Dio.netty.tryReflectionSetAccessible=true"
+    # --add-opens does not exist on jdk8
+    if [ "$USING_JDK8" -eq "0" ]; then
+      # Enable java.nio.DirectByteBuffer
+      # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
+      # https://github.com/netty/netty/issues/12265
+      NETTY_OPTS="$NETTY_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+    fi
+    echo $NETTY_OPTS
 }
 
 build_logging_opts() {
@@ -298,7 +306,15 @@ build_cli_logging_opts() {
 }
 
 build_bookie_opts() {
-  echo "-Djava.net.preferIPv4Stack=true"
+  BOOKIE_OPTS="-Djava.net.preferIPv4Stack=true"
+  # --add-opens does not exist on jdk8
+  if [ "$USING_JDK8" -eq "0" ]; then
+    # enable posix_fadvise usage in the Journal
+    BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.io=ALL-UNNAMED"
+    # DirectMemoryCRC32Digest
+    BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
+  fi
+  echo $BOOKIE_OPTS
 }
 
 find_table_service() {

pom.xml

@@ -1220,6 +1220,7 @@
           --add-opens java.base/java.util.concurrent=ALL-UNNAMED
           --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
           --add-opens java.base/java.util.stream=ALL-UNNAMED
+          --add-opens java.base/java.util.zip=ALL-UNNAMED
           --add-opens java.base/java.time=ALL-UNNAMED
           --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
           --add-opens java.base/sun.net.dns=ALL-UNNAMED

tests/scripts/src/test/bash/bk_test_bin_common.sh

@@ -226,9 +226,14 @@ testBuildNettyOpts() {
   source ${BK_BINDIR}/common.sh
 
   ACTUAL_NETTY_OPTS=$(build_netty_opts)
-  EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled"
+  EXPECTED_NETTY_OPTS=""
+  if [ "$USING_JDK8" -ne "1" ]; then
+      EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+  else
+      EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true"
+  fi
 
-    assertEquals "Netty OPTS is not set correctly" "${EXPECTED_NETTY_OPTS}" "${ACTUAL_NETTY_OPTS}"
+  assertEquals "Netty OPTS is not set correctly" "${EXPECTED_NETTY_OPTS}" "${ACTUAL_NETTY_OPTS}"
 }
 
 testBuildBookieOpts() {
@@ -237,6 +242,13 @@ testBuildBookieOpts() {
   ACTUAL_OPTS=$(build_bookie_opts)
   EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"
 
+  USEJDK8=$(detect_jdk8)
+  if [ "$USING_JDK8" -ne "1" ]; then
+    EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
+  else
+    EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"
+  fi
+
   assertEquals "Bookie OPTS is not set correctly" "${EXPECTED_OPTS}" "${ACTUAL_OPTS}"
 }
 

tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh

@@ -203,7 +203,12 @@ testBuildNettyOpts() {
   source ${BK_BINDIR}/common_gradle.sh
 
   ACTUAL_NETTY_OPTS=$(build_netty_opts)
-  EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled"
+  EXPECTED_NETTY_OPTS=""
+  if [ "$USING_JDK8" -ne "1" ]; then
+      EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+  else
+      EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true"
+  fi
 
     assertEquals "Netty OPTS is not set correctly" "${EXPECTED_NETTY_OPTS}" "${ACTUAL_NETTY_OPTS}"
 }
@@ -212,7 +217,12 @@ testBuildBookieOpts() {
   source ${BK_BINDIR}/common_gradle.sh
 
   ACTUAL_OPTS=$(build_bookie_opts)
-  EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"
+  USEJDK8=$(detect_jdk8)
+  if [ "$USING_JDK8" -ne "1" ]; then
+    EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
+  else
+    EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"
+  fi
 
   assertEquals "Bookie OPTS is not set correctly" "${EXPECTED_OPTS}" "${ACTUAL_OPTS}"
 }