[DNM] Maven release validation test

.github/workflows/maven_artifact_validation.yml

@@ -0,0 +1,84 @@
+name: Maven Release Artifact Validation
+
+on:
+  push:
+    branches:
+      - 'release-*'
+  pull_request:
+    branches:
+      - 'release-*'
+
+env:
+  MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5
+  SPARK_COMMON_MODULES: hudi-spark-datasource/hudi-spark,hudi-spark-datasource/hudi-spark-common
+
+jobs:
+  validate-release-maven-artifacts:
+    runs-on: ubuntu-latest
+    env:
+      HUDI_VERSION: 1.0.0
+      MAVEN_BASE_URL: 'https://repo1.maven.org/maven2'
+    strategy:
+      matrix:
+        include:
+          - scalaProfile: 'scala-2.13'
+            flinkProfile: 'flink1.20'
+            sparkProfile: 'spark3.5'
+            sparkRuntime: 'spark3.5.1'
+          - scalaProfile: 'scala-2.13'
+            flinkProfile: 'flink1.19'
+            sparkProfile: 'spark3.5'
+            sparkRuntime: 'spark3.5.1'
+          - scalaProfile: 'scala-2.13'
+            flinkProfile: 'flink1.18'
+            sparkProfile: 'spark3.5'
+            sparkRuntime: 'spark3.5.1'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.17'
+            sparkProfile: 'spark3.5'
+            sparkRuntime: 'spark3.5.1'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.16'
+            sparkProfile: 'spark3.4'
+            sparkRuntime: 'spark3.4.3'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.15'
+            sparkProfile: 'spark3.3'
+            sparkRuntime: 'spark3.3.4'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.14'
+            sparkProfile: 'spark3.3'
+            sparkRuntime: 'spark3.3.4'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          architecture: x64
+          cache: maven
+      - name: IT - Bundle Validation - OpenJDK 8
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java8 $HUDI_VERSION openjdk8 "" $MAVEN_BASE_URL
+      - name: IT - Bundle Validation - OpenJDK 11
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java11 $HUDI_VERSION openjdk11 "" $MAVEN_BASE_URL
+      - name: IT - Bundle Validation - OpenJDK 17
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 "" $MAVEN_BASE_URL

packaging/bundle-validation/README.md

@@ -19,7 +19,7 @@
 
 # Bundle Validation for Hudi
 
-This directory contains scripts for running bundle validation in Github Actions (`validate-bundles`
+This directory contains scripts for running bundle validation in GitHub Actions (`validate-bundles`
 specified in `.github/workflows/bot.yml`) and build profile for Docker images used.
 
 ## Docker Image for Bundle Validation
@@ -60,7 +60,7 @@ to `base/` and the image should only be used for development only and not be pus
 
 ## Running Bundle Validation on a Release Candidate
 
-The bundle validation on a release candidate is specified in the Github Action job `validate-release-candidate-bundles`
+The bundle validation on a release candidate is specified in the GitHub Action job `validate-release-candidate-bundles`
 in `.github/workflows/bot.yml`. By default, this is disabled.
 
 To enable the bundle validation on a particular release candidate, makes the following changes to the job by flipping the
@@ -73,3 +73,22 @@ env:
   STAGING_REPO_NUM: 1123
 ```
 
+## [Running Bundle Validation on Release Artifacts in Maven Central](#running-bundle-validation-on-release-artifacts-in-maven-central)
+
+After the release candidate artifacts are finalized and released from the staging repository, the artifacts usually take
+24 hours to be available in [Maven Central](https://repo1.maven.org/maven2/org/apache/hudi). The bundle validation can
+be run on the release artifacts in Maven Central by specifying the version in the GitHub Action job
+`validate-release-maven-artifacts` in `.github/workflows/maven_artifact_validation.yml`. By default, this is
+disabled.
+
+To enable the bundle validation on a particular release version, make the following changes to the job by flipping the
+flag and adding the release version:
+
+```yaml
+  validate-release-maven-artifacts:
+    runs-on: ubuntu-latest
+    if: false             ## Change to true, or simply remove/comment this line
+    env:
+      HUDI_VERSION: 1.0.0 ## Change to the release version
+      MAVEN_BASE_URL: 'https://repo1.maven.org/maven2'  ## Maven Central URL, no need to change unless necessary
+```

packaging/bundle-validation/ci_run.sh

@@ -31,9 +31,26 @@ CONTAINER_NAME=$1
 HUDI_VERSION=$2
 JAVA_RUNTIME_VERSION=$3
 STAGING_REPO_NUM=$4
+MAVEN_BASE_URL=$5
 echo "HUDI_VERSION: $HUDI_VERSION JAVA_RUNTIME_VERSION: $JAVA_RUNTIME_VERSION"
 echo "SPARK_RUNTIME: $SPARK_RUNTIME SPARK_PROFILE (optional): $SPARK_PROFILE"
 echo "SCALA_PROFILE: $SCALA_PROFILE"
+echo "MAVEN_BASE_URL: $MAVEN_BASE_URL"
+echo "STAGING_REPO_NUM: $STAGING_REPO_NUM"
+
+# Ensure only one of STAGING_REPO_NUM or MAVEN_BASE_URL is provided
+if [[ -n "$STAGING_REPO_NUM" && -n "$MAVEN_BASE_URL" ]]; then
+  echo "Error: Both STAGING_REPO_NUM and MAVEN_BASE_URL cannot be provided simultaneously."
+  exit 1
+fi
+
+if [[ -n "$STAGING_REPO_NUM" ]]; then
+  REPO_BASE_URL=https://repository.apache.org/content/repositories/orgapachehudi-$STAGING_REPO_NUM/org/apache/hudi
+  echo "Downloading bundle jars from staging repo orgapachehudi-$REPO_BASE_URL ..."
+elif [[ -n "$MAVEN_BASE_URL" ]]; then
+  REPO_BASE_URL=$MAVEN_BASE_URL/org/apache/hudi
+  echo "Downloading bundle jars from maven central - $REPO_BASE_URL ..."
+fi
 
 # choose versions based on build profiles
 if [[ ${SPARK_RUNTIME} == 'spark3.3.1' ]]; then
@@ -112,7 +129,7 @@ fi
 TMP_JARS_DIR=/tmp/jars/$(date +%s)
 mkdir -p $TMP_JARS_DIR
 
-if [[ -z "$STAGING_REPO_NUM" ]]; then
+if [ -z "$STAGING_REPO_NUM" ] && [ -z "$MAVEN_BASE_URL" ]; then
   echo 'Adding built bundle jars for validation'
   if [[ "$SCALA_PROFILE" != 'scala-2.13' ]]; then
     # For Scala 2.13, Flink is not support, so skipping the Flink bundle validation
@@ -126,7 +143,7 @@ if [[ -z "$STAGING_REPO_NUM" ]]; then
   cp ${GITHUB_WORKSPACE}/packaging/hudi-utilities-slim-bundle/target/hudi-*-$HUDI_VERSION.jar $TMP_JARS_DIR/
   echo 'Validating jars below:'
 else
-  echo 'Adding environment variables for bundles in the release candidate'
+  echo 'Adding environment variables for bundles in the release candidate or artifact'
 
   HUDI_HADOOP_MR_BUNDLE_NAME=hudi-hadoop-mr-bundle
   HUDI_KAFKA_CONNECT_BUNDLE_NAME=hudi-kafka-connect-bundle
@@ -170,8 +187,7 @@ else
     HUDI_FLINK_BUNDLE_NAME=hudi-flink1.20-bundle    
   fi
 
-  echo "Downloading bundle jars from staging repo orgapachehudi-$STAGING_REPO_NUM ..."
-  REPO_BASE_URL=https://repository.apache.org/content/repositories/orgapachehudi-$STAGING_REPO_NUM/org/apache/hudi
+  echo "Downloading bundle jars from base URL - $REPO_BASE_URL ..."
   wget -q $REPO_BASE_URL/$HUDI_FLINK_BUNDLE_NAME/$HUDI_VERSION/$HUDI_FLINK_BUNDLE_NAME-$HUDI_VERSION.jar -P $TMP_JARS_DIR/
   wget -q $REPO_BASE_URL/$HUDI_HADOOP_MR_BUNDLE_NAME/$HUDI_VERSION/$HUDI_HADOOP_MR_BUNDLE_NAME-$HUDI_VERSION.jar -P $TMP_JARS_DIR/
   wget -q $REPO_BASE_URL/$HUDI_KAFKA_CONNECT_BUNDLE_NAME/$HUDI_VERSION/$HUDI_KAFKA_CONNECT_BUNDLE_NAME-$HUDI_VERSION.jar -P $TMP_JARS_DIR/

release/release_guide.md

@@ -581,7 +581,15 @@ Once the release candidate has been reviewed and approved by the community, the
    > Note: we should close jira and choose 'resolution = Fixed' rather than resolve jira.
 9. Finalize the Release in Jira by providing the release date. 
 10. Update [DOAP](https://github.com/apache/hudi/blob/master/doap_HUDI.rdf) file in the root of the project via sending a PR like [this one](https://github.com/apache/incubator-hudi/pull/1448). 
-11. Create a new Github release, off the release version tag, you pushed before
+11. Create a new GitHub release, off the release version tag, you pushed before.
+
+After the release candidate artifacts are finalized and released from the staging repository, the artifacts usually take
+24 hours to be available in [Maven Central](https://repo1.maven.org/maven2/org/apache/hudi). Once the artifacts are
+available in Maven Central, please run the bundle validation GitHub Action to ensure the artifacts are valid. Steps to
+run the bundle validation GitHub Action are available in
+the [bundle validation documentation](../packaging/bundle-validation/README.md#running-bundle-validation-on-release-artifacts-in-maven-central).
+
+In the meantime, you can proceed with documentation changes and website updates.
 
 ## Steps to cut doc version and update website.