[improve][build] PIP-156 Build Pulsar Server on Java 17

.asf.yaml

@@ -58,14 +58,15 @@ github:
           - CI - Unit - Brokers - Broker Group 3
           - CI - Unit - Brokers - Client Api
           - CI - Unit - Brokers - Client Impl
-          - CI - Unit - Brokers - JDK 8
           - CI - Unit - Other
           - CI - Unit - Proxy
           - Build Pulsar java-test-image docker image
           - CI - Integration - Backwards Compatibility
           - CI - Integration - Cli
           - CI - Integration - Messaging
-          - CI - Integration - Shade
+          - CI - Integration - Shade on Java 8
+          - CI - Integration - Shade on Java 11
+          - CI - Integration - Shade on Java 17
           - CI - Integration - Standalone
           - CI - Integration - Transaction
           - Build Pulsar docker image

.github/workflows/pulsar-ci.yaml

@@ -150,9 +150,6 @@ jobs:
             group: BROKER_GROUP_2
           - name: Brokers - Broker Group 3
             group: BROKER_GROUP_3
-          - name: Brokers - JDK 8
-            group: BROKER_JDK8
-            jdk: 8
           - name: Brokers - Client Api
             group: BROKER_CLIENT_API
           - name: Brokers - Client Impl
@@ -347,8 +344,19 @@ jobs:
           - name: Messaging
             group: MESSAGING
 
-          - name: Shade
-            group: SHADE
+          - name: Shade on Java 8
+            group: SHADE_RUN
+            runtime_jdk: 8
+            setup: ./build/run_integration_group.sh SHADE_BUILD
+
+          - name: Shade on Java 11
+            group: SHADE_RUN
+            runtime_jdk: 11
+            setup: ./build/run_integration_group.sh SHADE_BUILD
+
+          - name: Shade on Java 17
+            group: SHADE_RUN
+            setup: ./build/run_integration_group.sh SHADE_BUILD
 
           - name: Standalone
             group: STANDALONE
@@ -410,6 +418,13 @@ jobs:
         run: |
           ${{ matrix.setup }}
 
+      - name: Set up runtime JDK ${{ matrix.runtime_jdk }}
+        uses: actions/setup-java@v2
+        if: ${{ matrix.runtime_jdk && needs.changed_files_job.outputs.docs_only != 'true' }}
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.runtime_jdk }}
+
       - name: Run integration test group '${{ matrix.group }}'
         if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
         run: |

README.md

@@ -83,10 +83,37 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la
 
 - [Pulsar CI](https://github.com/apache/pulsar-test-infra)
 
+## Pulsar Runtime Java Version Recommendation
+
+- pulsar master branch
+
+| Pulsar Components | Java Version  |
+| ----------------- |:-------------:|
+| Broker            | 17            |
+| Functions / IO    | 17            |
+| CLI               | 17            |
+| Java Client       | 8 or 11 or 17 |
+
+- 2.8 <= pulsar ver <= 2.10
+
+| Pulsar Components | Java Version  |
+| ----------------- |:-------------:|
+| Broker            | 11            |
+| Functions / IO    | 11            |
+| CLI               | 8 or 11       |
+| Java Client       | 8 or 11       |
+
+- pulsar ver 2.8 <
+
+| Pulsar Components | Java Version  |
+| ----------------- |:-------------:|
+| All               | 8 or 11       |
+
+
 ## Build Pulsar
 
 Requirements:
- * Java [JDK 11](https://adoptium.net/?variant=openjdk11) or [JDK 8](https://adoptium.net/?variant=openjdk8)
+ * Java [JDK 17](https://adoptium.net/?variant=openjdk17)
  * Maven 3.6.1+
  * zip
 
@@ -136,7 +163,8 @@ Check https://pulsar.apache.org for documentation and examples.
 
 Docker images must be built with Java 8 for `branch-2.7` or previous branches because of
 [issue 8445](https://github.com/apache/pulsar/issues/8445).
-Java 11 is the recommended JDK version in `master`/`branch-2.8`.
+Java 11 is the recommended JDK version in `branch-2.8`, `branch-2.9` and `branch-2.10`.
+Java 17 is the recommended JDK version in `master`.
 
 This builds the docker images `apachepulsar/pulsar-all:latest` and `apachepulsar/pulsar:latest`.
 
@@ -168,17 +196,17 @@ required plugins.
 
 ### Intellij
 
-#### Configure Project JDK to Java 11 JDK
+#### Configure Project JDK to Java 17 JDK
 
 1. Open **Project Settings**. 
 
     Click **File** -> **Project Structure** -> **Project Settings** -> **Project**.
 
 2. Select the JDK version.
 
-    From the JDK version drop-down list, select **Download JDK...** or choose an existing recent Java 11 JDK version.
+    From the JDK version drop-down list, select **Download JDK...** or choose an existing recent Java 17 JDK version.
 
-3. In the download dialog, select version **11**. You can pick a version from many vendors. Unless you have a specific preference, choose **Eclipse Temurin (AdoptOpenJDK (Hotspot))**.
+3. In the download dialog, select version **17**. You can pick a version from many vendors. Unless you have a specific preference, choose **Eclipse Temurin (AdoptOpenJDK (Hotspot))**.
 
 
 #### Configure Java version for Maven in IntelliJ

bouncy-castle/pom.xml

@@ -29,6 +29,20 @@
     <relativePath>..</relativePath>
   </parent>
 
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <configuration>
+            <release>${pulsar.client.compiler.release}</release>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
   <artifactId>bouncy-castle-parent</artifactId>
   <name>Apache Pulsar :: Bouncy Castle :: Parent</name>
 

build/run_integration_group.sh

@@ -55,6 +55,12 @@ mvn_run_integration_test() {
     RETRY=""
     shift
   fi
+  # skip test run if next parameter is "--build-only"
+  build_only=0
+  if [[ "$1" == "--build-only" ]]; then
+      build_only=1
+      shift
+  fi
   skip_build_deps=0
   while [[ "$1" == "--skip-build-deps" ]]; do
     skip_build_deps=1
@@ -69,19 +75,29 @@ mvn_run_integration_test() {
     mvn -B -T 1C -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dmaven.test.skip=true -Dcheckstyle.skip=true -Drat.skip=true -am install "$@"
     echo "::endgroup::"
   fi
-  echo "::group::Run tests for " "$@"
-  # use "verify" instead of "test"
-  $RETRY mvn -B -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DredirectTestOutputToFile=false verify "$@"
-  echo "::endgroup::"
-  set +x
-  "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports
+  if [[ $build_only -ne 1 ]]; then
+    echo "::group::Run tests for " "$@"
+    # use "verify" instead of "test"
+    $RETRY mvn -B -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DredirectTestOutputToFile=false verify "$@"
+    echo "::endgroup::"
+    set +x
+    "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports
+  fi
   )
 }
 
 test_group_shade() {
   mvn_run_integration_test "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false
 }
 
+test_group_shade_build() {
+  mvn_run_integration_test --build-only "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false
+}
+
+test_group_shade_run() {
+  mvn_run_integration_test --skip-build-deps "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false -Dmaven.compiler.source=8 -Dmaven.compiler.target=8
+}
+
 test_group_backwards_compat() {
   mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-backwards-compatibility.xml -DintegrationTests
   mvn_run_integration_test --retry "$@" -DBackwardsCompatTests -DtestForkCount=1 -DtestReuseFork=false

build/run_unit_group.sh

@@ -78,10 +78,6 @@ function test_group_broker_client_impl() {
   mvn_test -pl pulsar-broker -Dgroups='broker-impl'
 }
 
-function test_group_broker_jdk8() {
-  mvn_test -pl pulsar-broker -Dgroups='broker-jdk8' -Dpulsar.allocator.pooled=true
-}
-
 # prints summaries of failed tests to console
 # by using the targer/surefire-reports files
 # works only when testForkCount > 1 since that is when surefire will create reports for individual test classes

pom.xml

@@ -77,8 +77,9 @@ flexible messaging model and an intuitive client API.</description>
   </issueManagement>
 
   <properties>
-    <maven.compiler.source>8</maven.compiler.source>
-    <maven.compiler.target>8</maven.compiler.target>
+    <maven.compiler.source>17</maven.compiler.source>
+    <maven.compiler.target>17</maven.compiler.target>
+    <pulsar.client.compiler.release>8</pulsar.client.compiler.release>
 
     <!--config keys to configure test selection -->
     <include>**/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java</include>

pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java

@@ -34,7 +34,7 @@
  * ManagedLedgerInfo compression configuration test.
  */
 
-@Test(groups = {"broker", "broker-jdk8"})
+@Test(groups = {"broker"})
 public class ManagedLedgerCompressionTest extends BrokerTestBase {
 
     @BeforeClass

pulsar-client-1x-base/pom.xml

@@ -40,6 +40,13 @@
 
   <build>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <release>${pulsar.client.compiler.release}</release>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>com.github.spotbugs</groupId>
         <artifactId>spotbugs-maven-plugin</artifactId>

pulsar-client-admin-api/pom.xml

@@ -47,6 +47,13 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <release>${pulsar.client.compiler.release}</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.gaul</groupId>
                 <artifactId>modernizer-maven-plugin</artifactId>

pulsar-client-admin/pom.xml

@@ -116,6 +116,13 @@
 
   <build>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <release>${pulsar.client.compiler.release}</release>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.gaul</groupId>
         <artifactId>modernizer-maven-plugin</artifactId>

pulsar-client-all/pom.xml

@@ -134,7 +134,6 @@
               <artifactSet>
                 <includes>
                   <include>org.apache.pulsar:pulsar-client-original</include>
-                  <include>org.apache.pulsar:pulsar-transaction-common</include>
                   <include>org.apache.pulsar:pulsar-client-admin-original</include>
                   <include>org.apache.commons:commons-lang3</include>
                   <include>commons-codec:commons-codec</include>

pulsar-client-api/pom.xml

@@ -51,6 +51,13 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <release>${pulsar.client.compiler.release}</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.gaul</groupId>
                 <artifactId>modernizer-maven-plugin</artifactId>

pulsar-client-cpp/docker-tests.sh

@@ -65,7 +65,10 @@ DISABLE_COLOR_OUTPUT=""
 if [ "$GTEST_COLOR" = "no" ]; then
   DISABLE_COLOR_OUTPUT="| cat"
 fi
-$DOCKER_CMD bash -c "set -o pipefail; cd /pulsar/pulsar-client-cpp && ./run-unit-tests.sh ${tests} $DISABLE_COLOR_OUTPUT"
+
+# Java17 is required for CLI e.g) bin/pulsar create-token
+$DOCKER_CMD bash -c "apt-get -y install openjdk-17-jre-headless &&\
+ set -o pipefail; cd /pulsar/pulsar-client-cpp && ./run-unit-tests.sh ${tests} $DISABLE_COLOR_OUTPUT"
 RES=$?
 if [ $RES -ne 0 ]; then
   (

pulsar-client-messagecrypto-bc/pom.xml

@@ -53,6 +53,13 @@
 
   <build>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <release>${pulsar.client.compiler.release}</release>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-checkstyle-plugin</artifactId>

pulsar-client-shaded/pom.xml

@@ -129,7 +129,6 @@
               <artifactSet>
                 <includes>
                   <include>org.apache.pulsar:pulsar-client-original</include>
-                  <include>org.apache.pulsar:pulsar-transaction-common</include>
                   <include>org.apache.bookkeeper:*</include>
                   <include>org.apache.commons:commons-lang3</include>
                   <include>commons-codec:commons-codec</include>

pulsar-client/pom.xml

@@ -45,12 +45,6 @@
       <version>${project.parent.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>pulsar-transaction-common</artifactId>
-      <version>${project.parent.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>bouncy-castle-bc</artifactId>
@@ -195,6 +189,13 @@
     </resources>
     <!-- Generate protobuf for testing purposes -->
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <release>${pulsar.client.compiler.release}</release>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.gaul</groupId>
         <artifactId>modernizer-maven-plugin</artifactId>

pulsar-common/pom.xml

@@ -210,6 +210,14 @@
 
   <build>
   <plugins>
+    <plugin>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-compiler-plugin</artifactId>
+      <configuration>
+        <release>${pulsar.client.compiler.release}</release>
+      </configuration>
+    </plugin>
+
     <plugin>
       <groupId>org.gaul</groupId>
       <artifactId>modernizer-maven-plugin</artifactId>

site2/docs/deploy-bare-metal.md

@@ -40,7 +40,7 @@ To run Pulsar on bare metal, the following configuration is recommended:
 > * Broker is only supported on 64-bit JVM.
 > * If you do not have enough machines, or you want to test Pulsar in cluster mode (and expand the cluster later), You can fully deploy Pulsar on a node on which ZooKeeper, bookie and broker run.
 > * If you do not have a DNS server, you can use the multi-host format in the service URL instead.
-Each machine in your cluster needs to have [Java 8](https://adoptium.net/?variant=openjdk8), [Java 11](https://adoptium.net/?variant=openjdk11) or [Java 17](https://adoptium.net/?variant=openjdk17) installed.
+> * Each machine in your cluster needs to have the recommended Java version installed (e.g. [Java 17](https://adoptium.net/?variant=openjdk17)). Please refer to [Pulsar Runtime Java Version Recommendation](https://github.com/apache/pulsar/blob/master/README.md#pulsar-runtime-java-version-recommendation) according to your target Pulsar version.
 
 The following is a diagram showing the basic setup: