Skip to content

Enable Netty and BookKeeper IO optimizations on jdk17 #3234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merlimat merged 2 commits into from
May 11, 2022
Merged

Enable Netty and BookKeeper IO optimizations on jdk17 #3234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
acaa135
Enable netty optimization and BK native IO on jdk17
nicoloboschi Apr 1, 2022
967e159
zip and better docs
nicoloboschi Apr 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions bin/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,15 @@ build_cli_jvm_opts() {
}

build_netty_opts() {
echo "-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL}"
NETTY_OPTS="-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} -Dio.netty.tryReflectionSetAccessible=true"
# --add-opens does not exist on jdk8
if [ "$USING_JDK8" -eq "0" ]; then
# Enable java.nio.DirectByteBuffer
# https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
# https://github.com/netty/netty/issues/12265
NETTY_OPTS="$NETTY_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
fi
echo $NETTY_OPTS
}

build_logging_opts() {
Expand Down Expand Up @@ -312,7 +320,15 @@ build_cli_logging_opts() {
}

build_bookie_opts() {
echo "-Djava.net.preferIPv4Stack=true"
BOOKIE_OPTS="-Djava.net.preferIPv4Stack=true"
# --add-opens does not exist on jdk8
if [ "$USING_JDK8" -eq "0" ]; then
# enable posix_fadvise usage in the Journal
BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.io=ALL-UNNAMED"
# DirectMemoryCRC32Digest
BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
fi
echo $BOOKIE_OPTS
}

find_table_service() {
Expand Down
20 changes: 18 additions & 2 deletions bin/common_gradle.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,15 @@ build_cli_jvm_opts() {
}

build_netty_opts() {
echo "-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL}"
NETTY_OPTS="-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} -Dio.netty.tryReflectionSetAccessible=true"
# --add-opens does not exist on jdk8
if [ "$USING_JDK8" -eq "0" ]; then
# Enable java.nio.DirectByteBuffer
# https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
# https://github.com/netty/netty/issues/12265
NETTY_OPTS="$NETTY_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
fi
echo $NETTY_OPTS
}

build_logging_opts() {
Expand Down Expand Up @@ -294,7 +302,15 @@ build_cli_logging_opts() {
}

build_bookie_opts() {
echo "-Djava.net.preferIPv4Stack=true"
BOOKIE_OPTS="-Djava.net.preferIPv4Stack=true"
# --add-opens does not exist on jdk8
if [ "$USING_JDK8" -eq "0" ]; then
# enable posix_fadvise usage in the Journal
BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.io=ALL-UNNAMED"
# DirectMemoryCRC32Digest
BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
fi
echo $BOOKIE_OPTS
}

find_table_service() {
Expand Down
1 change: 1 addition & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1226,6 +1226,7 @@
--add-opens java.base/java.util=ALL-UNNAMED
--add-opens java.base/java.util.concurrent=ALL-UNNAMED
--add-opens java.base/java.util.stream=ALL-UNNAMED
--add-opens java.base/java.util.zip=ALL-UNNAMED
--add-opens java.base/java.time=ALL-UNNAMED
--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
--add-opens java.base/sun.net.dns=ALL-UNNAMED
Expand Down
16 changes: 14 additions & 2 deletions tests/scripts/src/test/bash/bk_test_bin_common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,9 +228,14 @@ testBuildNettyOpts() {
source ${BK_BINDIR}/common.sh

ACTUAL_NETTY_OPTS=$(build_netty_opts)
EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled"
EXPECTED_NETTY_OPTS=""
if [ "$USING_JDK8" -ne "1" ]; then
EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
else
EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true"
fi

assertEquals "Netty OPTS is not set correctly" "${EXPECTED_NETTY_OPTS}" "${ACTUAL_NETTY_OPTS}"
assertEquals "Netty OPTS is not set correctly" "${EXPECTED_NETTY_OPTS}" "${ACTUAL_NETTY_OPTS}"
}

testBuildBookieOpts() {
Expand All @@ -239,6 +244,13 @@ testBuildBookieOpts() {
ACTUAL_OPTS=$(build_bookie_opts)
EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"

USEJDK8=$(detect_jdk8)
if [ "$USING_JDK8" -ne "1" ]; then
EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
else
EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"
fi

assertEquals "Bookie OPTS is not set correctly" "${EXPECTED_OPTS}" "${ACTUAL_OPTS}"
}

Expand Down
14 changes: 12 additions & 2 deletions tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,12 @@ testBuildNettyOpts() {
source ${BK_BINDIR}/common_gradle.sh

ACTUAL_NETTY_OPTS=$(build_netty_opts)
EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled"
EXPECTED_NETTY_OPTS=""
if [ "$USING_JDK8" -ne "1" ]; then
EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
else
EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled -Dio.netty.tryReflectionSetAccessible=true"
fi

assertEquals "Netty OPTS is not set correctly" "${EXPECTED_NETTY_OPTS}" "${ACTUAL_NETTY_OPTS}"
}
Expand All @@ -214,7 +219,12 @@ testBuildBookieOpts() {
source ${BK_BINDIR}/common_gradle.sh

ACTUAL_OPTS=$(build_bookie_opts)
EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"
USEJDK8=$(detect_jdk8)
if [ "$USING_JDK8" -ne "1" ]; then
EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
else
EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true"
fi

assertEquals "Bookie OPTS is not set correctly" "${EXPECTED_OPTS}" "${ACTUAL_OPTS}"
}
Expand Down