[improve][misc] Sync commits from apache/pulsar branch-3.0 (30th June) into 3.1_ds

.github/workflows/ci-maven-cache-update.yaml

@@ -63,7 +63,7 @@ jobs:
             mvn_arguments: ''
 
           - name: all modules - macos
-            runs-on: macos-11
+            runs-on: macos-latest
             cache_name: 'm2-dependencies-all'
 
           - name: core-modules

.github/workflows/pulsar-ci.yaml

@@ -1209,7 +1209,7 @@ jobs:
 
   macos-build:
     name: Build Pulsar on MacOS
-    runs-on: macos-11
+    runs-on: macos-latest
     timeout-minutes: 120
     needs: ['preconditions', 'integration-tests']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}

.gitignore

@@ -97,3 +97,5 @@ test-reports/
 
 # Gradle Enterprise
 .mvn/.gradle-enterprise/
+# Gradle Develocity
+.mvn/.develocity/

.mvn/gradle-enterprise.xml → .mvn/develocity.xml

@@ -19,22 +19,21 @@
     under the License.
 
 -->
-<gradleEnterprise
-        xmlns="https://www.gradle.com/gradle-enterprise-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="https://www.gradle.com/gradle-enterprise-maven https://www.gradle.com/schema/gradle-enterprise-maven.xsd">
+<develocity xmlns="https://www.gradle.com/develocity-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:schemaLocation="https://www.gradle.com/develocity-maven https://www.gradle.com/schema/develocity-maven.xsd">
+<!-- Enable Gradle Develocity extension when GRADLE_ENTERPRISE_ACCESS_KEY/DEVELOCITY_ACCESS_KEY is set and the build isn't
+       a pull request from a branch or forked repository with a name that indicates it's a work in progress. -->
+  <enabled>#{(env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > '' or env['DEVELOCITY_ACCESS_KEY']?.trim() > '') and !(env['GITHUB_HEAD_REF']?.matches('(?i).*(experiment|wip|private).*') or env['GITHUB_REPOSITORY']?.matches('(?i).*(experiment|wip|private).*'))}</enabled>
   <server>
     <url>https://ge.apache.org</url>
     <allowUntrusted>false</allowUntrusted>
   </server>
   <buildScan>
     <capture>
-      <goalInputFiles>true</goalInputFiles>
       <buildLogging>true</buildLogging>
       <testLogging>true</testLogging>
     </capture>
     <backgroundBuildScanUpload>#{isFalse(env['GITHUB_ACTIONS'])}</backgroundBuildScanUpload>
-    <publish>ALWAYS</publish>
-    <publishIfAuthenticated>true</publishIfAuthenticated>
     <obfuscation>
       <ipAddresses>#{{'0.0.0.0'}}</ipAddresses>
     </obfuscation>
@@ -47,4 +46,4 @@
       <enabled>false</enabled>
     </remote>
   </buildCache>
-</gradleEnterprise>
+</develocity>

.mvn/extensions.xml

@@ -23,12 +23,12 @@
             xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
   <extension>
     <groupId>com.gradle</groupId>
-    <artifactId>gradle-enterprise-maven-extension</artifactId>
-    <version>1.17.1</version>
+    <artifactId>develocity-maven-extension</artifactId>
+    <version>1.21.4</version>
   </extension>
   <extension>
     <groupId>com.gradle</groupId>
     <artifactId>common-custom-user-data-maven-extension</artifactId>
-    <version>1.11.1</version>
+    <version>2.0</version>
   </extension>
 </extensions>

README.md

@@ -141,8 +141,6 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la
 >
 > This project includes a [Maven Wrapper](https://maven.apache.org/wrapper/) that can be used instead of a system-installed Maven.
 > Use it by replacing `mvn` by `./mvnw` on Linux and `mvnw.cmd` on Windows in the commands below.    
->
-> It's better to use CMD rather than Powershell on Windows. Because maven will activate the `windows` profile which runs `rename-netty-native-libs.cmd`.
 
 ### Build
 

bin/pulsar-daemon

@@ -157,7 +157,7 @@ start ()
     echo starting $command, logging to $logfile
     echo Note: Set immediateFlush to true in conf/log4j2.yaml will guarantee the logging event is flushing to disk immediately. The default behavior is switched off due to performance considerations.
     pulsar=$PULSAR_HOME/bin/pulsar
-    nohup $pulsar $command "$1" > "$out" 2>&1 < /dev/null &
+    nohup $pulsar $command "$@" > "$out" 2>&1 < /dev/null &
     echo $! > $pid
     sleep 1; head $out
     sleep 2;
@@ -216,29 +216,28 @@ stop ()
 
 case $startStop in
   (start)
-    start "$*"
+    start "$@"
     ;;
 
   (stop)
     stop $1
     ;;
 
   (restart)
-    forceStopFlag=$(echo "$*"|grep "\-force")
-    if [[ "$forceStopFlag" != "" ]]
+    if [[ "$1" == "-force" ]]
     then
-      stop "-force"
+      stop -force
+      # remove "-force" from the arguments
+      shift
     else
       stop
     fi
     if [ "$?" == 0 ]
     then
-       sleep 3
-       paramaters="$*"
-       startParamaters=${paramaters//-force/}
-       start "$startParamaters"
+      sleep 3
+      start "$@"
     else
-       echo "WARNNING :  $command failed restart, for $command is not stopped completely."
+      echo "WARNNING :  $command failed restart, for $command is not stopped completely."
     fi
     ;;
 

build/pulsar_ci_tool.sh

@@ -389,6 +389,7 @@ _ci_upload_coverage_files() {
                   --transform="flags=r;s|\\(/jacoco.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \
                   --transform="flags=r;s|\\(/tmp/jacocoDir/.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \
                   --exclude="*/META-INF/bundled-dependencies/*" \
+                  --exclude="*/META-INF/versions/*" \
                   $GITHUB_WORKSPACE/target/classpath_* \
                   $(find "$GITHUB_WORKSPACE" -path "*/target/jacoco*.exec" -printf "%p\n%h/classes\n" | sort | uniq) \
                   $([ -d /tmp/jacocoDir ] && echo "/tmp/jacocoDir" ) \
@@ -530,11 +531,11 @@ ci_create_test_coverage_report() {
     local classfilesArgs="--classfiles $({
       {
         for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; }); do
-            if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 2>/dev/null)" ]]; then
-              # file must be processed by removing META-INF/bundled-dependencies
+            if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' 2>/dev/null)" ]]; then
+              # file must be processed by removing META-INF/bundled-dependencies and META-INF/versions
               local jartempfile=$(mktemp -t jarfile.XXXX --suffix=.jar)
               cp $classpathEntry $jartempfile
-              zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' &> /dev/null
+              zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' &> /dev/null
               echo $jartempfile
             else
               echo $classpathEntry
@@ -596,7 +597,7 @@ ci_create_inttest_coverage_report() {
       # remove jar file that causes duplicate classes issue
       rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle* || true
       # remove any bundled dependencies as part of .jar/.nar files
-      find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' \; |grep -E -v "Nothing to do|^$" || true
+      find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' \; |grep -E -v "Nothing to do|^$" || true
     fi
     # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed
     local excludeProjectsPattern="testmocks|testclient|buildtools"

buildtools/pom.xml

@@ -47,7 +47,7 @@
     <license-maven-plugin.version>4.1</license-maven-plugin.version>
     <puppycrawl.checkstyle.version>10.14.2</puppycrawl.checkstyle.version>
     <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
-    <netty.version>4.1.108.Final</netty.version>
+    <netty.version>4.1.111.Final</netty.version>
     <guice.version>4.2.3</guice.version>
     <guava.version>32.1.1-jre</guava.version>
     <ant.version>1.10.12</ant.version>

distribution/server/pom.xml

@@ -46,6 +46,19 @@
       <version>${project.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>pulsar-metadata</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>jetcd-core-shaded</artifactId>
+      <version>${project.version}</version>
+      <classifier>shaded</classifier>
+    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-proxy</artifactId>

distribution/server/src/assemble/LICENSE.bin.txt

@@ -289,26 +289,26 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.11.jar
     - org.apache.commons-commons-text-1.10.0.jar
  * Netty
-    - io.netty-netty-buffer-4.1.108.Final.jar
-    - io.netty-netty-codec-4.1.108.Final.jar
-    - io.netty-netty-codec-dns-4.1.108.Final.jar
-    - io.netty-netty-codec-http-4.1.108.Final.jar
-    - io.netty-netty-codec-http2-4.1.108.Final.jar
-    - io.netty-netty-codec-socks-4.1.108.Final.jar
-    - io.netty-netty-codec-haproxy-4.1.108.Final.jar
-    - io.netty-netty-common-4.1.108.Final.jar
-    - io.netty-netty-handler-4.1.108.Final.jar
-    - io.netty-netty-handler-proxy-4.1.108.Final.jar
-    - io.netty-netty-resolver-4.1.108.Final.jar
-    - io.netty-netty-resolver-dns-4.1.108.Final.jar
-    - io.netty-netty-resolver-dns-classes-macos-4.1.108.Final.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.108.Final-osx-aarch_64.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.108.Final-osx-x86_64.jar
-    - io.netty-netty-transport-4.1.108.Final.jar
-    - io.netty-netty-transport-classes-epoll-4.1.108.Final.jar
-    - io.netty-netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar
-    - io.netty-netty-transport-native-unix-common-4.1.108.Final.jar
-    - io.netty-netty-transport-native-unix-common-4.1.108.Final-linux-x86_64.jar
+    - io.netty-netty-buffer-4.1.111.Final.jar
+    - io.netty-netty-codec-4.1.111.Final.jar
+    - io.netty-netty-codec-dns-4.1.111.Final.jar
+    - io.netty-netty-codec-http-4.1.111.Final.jar
+    - io.netty-netty-codec-http2-4.1.111.Final.jar
+    - io.netty-netty-codec-socks-4.1.111.Final.jar
+    - io.netty-netty-codec-haproxy-4.1.111.Final.jar
+    - io.netty-netty-common-4.1.111.Final.jar
+    - io.netty-netty-handler-4.1.111.Final.jar
+    - io.netty-netty-handler-proxy-4.1.111.Final.jar
+    - io.netty-netty-resolver-4.1.111.Final.jar
+    - io.netty-netty-resolver-dns-4.1.111.Final.jar
+    - io.netty-netty-resolver-dns-classes-macos-4.1.111.Final.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.111.Final-osx-aarch_64.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.111.Final-osx-x86_64.jar
+    - io.netty-netty-transport-4.1.111.Final.jar
+    - io.netty-netty-transport-classes-epoll-4.1.111.Final.jar
+    - io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar
+    - io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-unix-common-4.1.111.Final.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar
@@ -345,34 +345,34 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.5.jar
-    - org.apache.bookkeeper-circe-checksum-4.16.5.jar
-    - org.apache.bookkeeper-cpu-affinity-4.16.5.jar
-    - org.apache.bookkeeper-statelib-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-api-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-common-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-server-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.16.5.jar
-    - org.apache.bookkeeper.http-http-server-4.16.5.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.16.5.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.5.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-common-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.5-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.16.5.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.5.jar
-    - org.apache.bookkeeper-native-io-4.16.5.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.6.jar
+    - org.apache.bookkeeper-circe-checksum-4.16.6.jar
+    - org.apache.bookkeeper-cpu-affinity-4.16.6.jar
+    - org.apache.bookkeeper-statelib-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-api-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-common-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-server-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.16.6.jar
+    - org.apache.bookkeeper.http-http-server-4.16.6.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.16.6.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.6.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.6.jar
+    - org.apache.distributedlog-distributedlog-common-4.16.6.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.6-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.6.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.16.6.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.6.jar
+    - org.apache.bookkeeper-native-io-4.16.6.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
@@ -423,7 +423,6 @@ The Apache Software License, Version 2.0
     - io.grpc-grpc-auth-1.55.3.jar
     - io.grpc-grpc-context-1.55.3.jar
     - io.grpc-grpc-core-1.55.3.jar
-    - io.grpc-grpc-netty-1.55.3.jar
     - io.grpc-grpc-protobuf-1.55.3.jar
     - io.grpc-grpc-protobuf-lite-1.55.3.jar
     - io.grpc-grpc-stub-1.55.3.jar
@@ -482,7 +481,6 @@ The Apache Software License, Version 2.0
     - io.vertx-vertx-core-4.5.8.jar
     - io.vertx-vertx-web-4.5.8.jar
     - io.vertx-vertx-web-common-4.5.8.jar
-    - io.vertx-vertx-grpc-4.5.8.jar
   * Apache ZooKeeper
     - org.apache.zookeeper-zookeeper-3.9.2.jar
     - org.apache.zookeeper-zookeeper-jute-3.9.2.jar
@@ -494,17 +492,14 @@ The Apache Software License, Version 2.0
     - com.google.http-client-google-http-client-1.41.0.jar
     - com.google.auto.value-auto-value-annotations-1.9.jar
     - com.google.re2j-re2j-1.6.jar
-  * Jetcd
-    - io.etcd-jetcd-api-0.7.7.jar
-    - io.etcd-jetcd-common-0.7.7.jar
-    - io.etcd-jetcd-core-0.7.7.jar
-    - io.etcd-jetcd-grpc-0.7.7.jar
+  * Jetcd - shaded
   * IPAddress
-    - com.github.seancfoley-ipaddress-5.3.3.jar
+    - com.github.seancfoley-ipaddress-5.5.0.jar
   * RxJava
     - io.reactivex.rxjava3-rxjava-3.0.1.jar
   * RoaringBitmap
     - org.roaringbitmap-RoaringBitmap-0.9.44.jar
+    - org.roaringbitmap-shims-0.9.44.jar
 
 BSD 3-clause "New" or "Revised" License
  * Google auth library
@@ -537,7 +532,6 @@ Protocol Buffers License
 
 CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
  * Java Annotations API
-    - javax.annotation-javax.annotation-api-1.3.2.jar
     - com.sun.activation-javax.activation-1.2.0.jar
     - javax.xml.bind-jaxb-api-2.3.1.jar
  * Java Servlet API -- javax.servlet-javax.servlet-api-3.1.0.jar
@@ -607,6 +601,7 @@ Datastax
     - com.datastax.oss-pulsar-transaction-common-3.1.4.3-SNAPSHOT.jar
     - com.datastax.oss-pulsar-transaction-coordinator-3.1.4.3-SNAPSHOT.jar
     - com.datastax.oss-testmocks-3.1.4.3-SNAPSHOT.jar
+    - com.datastax.oss-jetcd-core-shaded-3.1.4.3-SNAPSHOT-shaded.jar
 
 ------------------------
 

distribution/server/src/assemble/bin.xml

@@ -126,21 +126,23 @@
     <dependencySet>
       <outputDirectory>lib</outputDirectory>
       <unpack>false</unpack>
-      <scope>compile</scope>
+      <scope>runtime</scope>
       <useProjectArtifact>false</useProjectArtifact>
       <!-- Include 'groupId' in the dependencies Jar names to better identify
            the provenance of the jar -->
       <outputFileNameMapping>${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension}</outputFileNameMapping>
 
       <excludes>
         <exclude>com.datastax.oss:pulsar-functions-runtime-all</exclude>
-
-        <exclude>org.projectlombok:lombok</exclude>
-
         <!-- prevent adding pulsar-functions-api-examples in lib -->
         <exclude>com.datastax.oss:pulsar-functions-api-examples</exclude>
         <!-- prevent adding any distribution .tar.gz files in lib -->
         <exclude>*:tar.gz</exclude>
+        <!-- prevent adding annotation libraries -->
+        <exclude>org.codehaus.mojo:animal-sniffer-annotations</exclude>
+        <exclude>com.google.android:annotations</exclude>
+        <!-- Needed only in the pulsar-shell distro only -->
+        <exclude>net.java.dev.jna:jna</exclude>
       </excludes>
     </dependencySet>
   </dependencySets>