[improve][misc] Sync commits from apache into 3.1_ds

.github/actions/clean-disk/action.yml

@@ -31,7 +31,7 @@ runs:
           directories=(/usr/local/lib/android /opt/ghc)
           if [[ "${{ inputs.mode }}" == "full" ]]; then
             # remove these directories only when mode is 'full'
-            directories+=(/usr/share/dotnet)
+            directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
           fi
           emptydir=/tmp/empty$$/
           mkdir $emptydir

.github/actions/tune-runner-vm/action.yml

@@ -53,8 +53,8 @@ runs:
             # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
             # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
             # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
             # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
             # https://www.spinics.net/lists/linux-ide/msg52562.html
             for i in /sys/block/sd*/queue/discard_max_bytes; do

.github/actions/upload-coverage/action.yml

@@ -51,7 +51,7 @@ runs:
     - name: "Upload to Codecov (attempt #1)"
       id: codecov-upload-1
       if: steps.repo-check.outputs.passed == 'true'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: ${{ inputs.flags }}
@@ -64,7 +64,7 @@ runs:
     - name: "Upload to Codecov (attempt #2)"
       id: codecov-upload-2
       if: steps.codecov-upload-1.outcome == 'failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: ${{ inputs.flags }}
@@ -77,7 +77,7 @@ runs:
     - name: "Upload to Codecov (attempt #3)"
       id: codecov-upload-3
       if: steps.codecov-upload-2.outcome == 'failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       # fail on last attempt
       continue-on-error: false
       with:
@@ -97,4 +97,4 @@ runs:
 
         [Code coverage report](https://app.codecov.io/github/$GITHUB_REPOSITORY/commit/${head_sha}/tree)
 
-        EOF
+        EOF

.github/changes-filter.yaml

@@ -12,10 +12,12 @@ docs:
   - 'deployment/**'
   - 'wiki/**'
   - 'pip/**'
+java_non_tests:
+  - '**/src/main/java/**/*.java'
 tests:
   - added|modified: '**/src/test/java/**/*.java'
 need_owasp:
   - 'pom.xml'
   - '**/pom.xml'
   - 'src/owasp-dependency-check-false-positives.xml'
-  - 'src/owasp-dependency-check-suppressions.xml'
+  - 'src/owasp-dependency-check-suppressions.xml'

.github/workflows/ci-go-functions.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - 3.1_ds
     paths:
       - '.github/workflows/**'
       - 'pulsar-function-go/**'
@@ -42,7 +43,7 @@ jobs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Detect changed files
         id: changes
@@ -75,25 +76,25 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        go-version: [1.15, 1.16, 1.17]
+        go-version: ['1.21']
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
         id: go
 
       - name: InstallTool
         run: |
           cd pulsar-function-go
-          wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.18.0
+          wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.55.2
           ./bin/golangci-lint --version
 
       - name: Build

.github/workflows/ci-maven-cache-update.yaml

@@ -63,7 +63,7 @@ jobs:
             mvn_arguments: ''
 
           - name: all modules - macos
-            runs-on: macos-11
+            runs-on: macos-latest
             cache_name: 'm2-dependencies-all'
 
           - name: core-modules

.github/workflows/ci-owasp-dependency-check.yaml

@@ -24,7 +24,9 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  JDK_DISTRIBUTION: corretto
+  NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
 
 jobs:
   run-owasp-dependency-check:
@@ -34,65 +36,96 @@ jobs:
       JOB_NAME: Check ${{ matrix.branch }}
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 45
+    timeout-minutes: 75
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         include:
           - branch: master
+          - branch: branch-3.3
+          - branch: branch-3.2
           - branch: branch-3.0
-          - branch: branch-2.11
-          - branch: branch-2.10
-            jdk: 11
-          - branch: branch-2.9
-            jdk: 11
-          - branch: branch-2.8
-            jdk: 11
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ matrix.branch }}
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v3
+      - name: Restore Maven repository cache
+        uses: actions/cache/restore@v4
         timeout-minutes: 5
         with:
           path: |
             ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/pulsar
-          key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
 
       - name: Set up JDK ${{ matrix.jdk || '17' }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
+          distribution: ${{ env.JDK_DISTRIBUTION }}
           java-version: ${{ matrix.jdk || '17' }}
 
       - name: run install by skip tests
-        run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true
+        run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders'
+
+      - name: OWASP cache key weeknum
+        id: get-weeknum
+        run: |
+          echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Restore OWASP Dependency Check data
+        id: restore-owasp-dependency-check-data
+        uses: actions/cache/restore@v4
+        timeout-minutes: 5
+        with:
+          path: ~/.m2/repository/org/owasp/dependency-check-data
+          key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
+          enableCrossOsArchive: true
+          restore-keys: |
+            owasp-dependency-check-data-
+
+      - name: Update OWASP Dependency Check data
+        id: update-owasp-dependency-check-data
+        if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }}
+        run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only
+
+      - name: Save OWASP Dependency Check data
+        if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }}
+        uses: actions/cache/save@v4
+        timeout-minutes: 5
+        with:
+          path: ~/.m2/repository/org/owasp/dependency-check-data
+          key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }}
+          enableCrossOsArchive: true
 
       - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true)
         run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true
 
-      - name: run OWASP Dependency Check for distribution/offloaders, distribution/io and pulsar-sql/presto-distribution
-        run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io,pulsar-sql/presto-distribution
+      - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false)
+        if: ${{ !cancelled() }}
+        run: |
+          mvnprojects=$(mvn -B -ntp -Dscan=false initialize \
+            | grep -- "-< .* >-" \
+            | sed -E 's/.*-< (.*) >-.*/\1/' \
+            | grep -E 'pulsar-io-|tiered-storage-|offloader' \
+            | tr '\n' ',' | sed 's/,$/\n/' )
+          set -xe
+          mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}"
 
       - name: Upload OWASP Dependency Check reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: owasp-dependency-check-reports-${{ matrix.branch }}
           path: |
-            distribution/server/target/dependency-check-report.html
-            distribution/offloaders/target/dependency-check-report.html
-            distribution/io/target/dependency-check-report.html
-            pulsar-sql/presto-distribution/target/dependency-check-report.html
+            **/target/dependency-check-report.html

.github/workflows/pulsar-ci-flaky.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - 3.1_ds
   schedule:
     - cron: '0 12 * * *'
   workflow_dispatch:
@@ -36,7 +37,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
@@ -53,7 +54,7 @@ jobs:
       collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Detect changed files
         id:   changes
@@ -100,7 +101,7 @@ jobs:
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
@@ -114,7 +115,7 @@ jobs:
           limit-access-to-actor: true
 
       - name: Cache local Maven repository
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         timeout-minutes: 5
         with:
           path: |
@@ -125,7 +126,7 @@ jobs:
             ${{ runner.os }}-m2-dependencies-core-modules-
 
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: 17
@@ -155,7 +156,7 @@ jobs:
 
       - name: Upload Jacoco report files to build artifacts
         if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Jacoco-coverage-report-flaky
           path: target/jacoco_test_coverage_report_flaky.zip
@@ -175,15 +176,15 @@ jobs:
           annotate_only: 'true'
 
       - name: Upload Surefire reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ !success() }}
         with:
           name: Unit-BROKER_FLAKY-surefire-reports
           path: surefire-reports
           retention-days: 7
 
       - name: Upload possible heap dump
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Unit-BROKER_FLAKY-heapdump