Upgrade xgboost to 3.0.4 (#3788)

* Upgrade xgboost to 3.0.4

* Remove gpu build

Author: xyang16

.github/workflows/native_s3_xgboost.yml

@@ -30,26 +30,24 @@ jobs:
 
   build-xgboost-jni-aarch64:
     runs-on: [ self-hosted, aarch64 ]
-    container: centos:7
     timeout-minutes: 30
     needs: create-aarch64-runner
+    container:
+      image: amazonlinux:2023
+      env:
+        JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto
     steps:
-      - uses: actions/checkout@v3
       - name: Install Environment
         run: |
-          yum -y update
-          yum -y install centos-release-scl-rh epel-release
-          yum -y install devtoolset-8 git patch libstdc++-static curl python3-devel
-          curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.27.0-rc2/cmake-3.27.0-rc2-linux-aarch64.tar.gz
-          tar xvfz cmake.tar.gz
-          ln -sf $PWD/cmake-3.*/bin/cmake /usr/bin/cmake
-          cmake --version
-          pip3 install awscli --upgrade
+          dnf -y group install "Development Tools"
+          dnf -y install patch git cmake3 python3-devel python3-pip java-17-amazon-corretto-devel jq
+          pip3 install awscli wheel build --upgrade
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
           distribution: 'corretto'
           java-version: 17
+      - uses: actions/checkout@v4
       - name: Release JNI prep
         run: |
           XGBOOST_VERSION=${{ github.event.inputs.xgb_version }}
@@ -60,17 +58,29 @@ jobs:
           python3 create_jni.py
           cd ../..
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role
-          aws-region: us-east-2
+        run: |
+          oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
+            "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value")
+          echo "::add-mask::$oidc_token"
+          read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \
+            --region "us-east-2" \
+            --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \
+            --role-session-name "build-tokenizer-jni-aarch64" \
+            --web-identity-token "$oidc_token" \
+            --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \
+            --output text)"
+          echo "::add-mask::$AWS_ACCESS_KEY_ID"
+          echo "::add-mask::$AWS_SECRET_ACCESS_KEY"
+          echo "::add-mask::$AWS_SESSION_TOKEN"
+          echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV"
+          echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV"
+          echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV"
       - name: Copy files to S3 with the AWS CLI
         run: |
           XGBOOST_VERSION=${{ github.event.inputs.xgb_version }}
-          XGBOOST_VERSION=${XGBOOST_VERSION:-$(awk -F '=' '/xgboost/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)')}
+          XGBOOST_VERSION=${XGBOOST_VERSION:-$(awk -F '=' '/xgboost/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)}
           aws s3 cp xgboost/lib/libxgboost4j.so s3://djl-ai/publish/xgboost/${XGBOOST_VERSION}/jnilib/linux/aarch64/
           aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/xgboost/${XGBOOST_VERSION}/jnilib*"
-
   stop-runners:
     if: ${{ github.repository == 'deepjavalibrary/djl' && always() }}
     runs-on: [ self-hosted, scheduler ]

.github/workflows/nightly_publish.yml

@@ -80,7 +80,7 @@ jobs:
         run: |
           ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.13.1 -Psnapshot
           ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.5.1 -Psnapshot
-          ./gradlew clean engines:ml:xgboost:publish -Pgpu -Psnapshot
+          ./gradlew clean engines:ml:xgboost:publish -Psnapshot
           ./gradlew clean publish -Psnapshot
           cd bom
           ./gradlew publish -Psnapshot
@@ -94,7 +94,7 @@ jobs:
         run: |
           ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.13.1 -P${{ github.event.inputs.mode }}
           ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.5.1 -P${{ github.event.inputs.mode }}
-          ./gradlew clean engines:ml:xgboost:publish -Pgpu -P${{ github.event.inputs.mode }}
+          ./gradlew clean engines:ml:xgboost:publish -P${{ github.event.inputs.mode }}
           ./gradlew clean publish -P${{ github.event.inputs.mode }}
           cd bom
           ./gradlew publish -P${{ github.event.inputs.mode }}

engines/ml/xgboost/build.gradle.kts

@@ -11,7 +11,6 @@ val xgbFlavor = if (isGpu) "-gpu" else ""
 
 val exclusion by configurations.registering
 
-@Suppress("UnstableApiUsage")
 dependencies {
     api(project(":api"))
     api(libs.commons.logging)
@@ -42,23 +41,6 @@ dependencies {
 tasks {
     compileJava { dependsOn(processResources) }
 
-    processResources {
-        val jnilibDir = buildDirectory / "resources/main/lib/linux/aarch64/"
-        inputs.properties(mapOf("xgboostVersion" to libs.versions.xgboost.get()))
-        outputs.dir(jnilibDir)
-
-        val logger = project.logger
-        doLast {
-            val url =
-                "https://publish.djl.ai/xgboost/${libs.versions.xgboost.get()}/jnilib/linux/aarch64/libxgboost4j.so"
-            val file = jnilibDir / "libxgboost4j.so"
-            if (!file.exists()) {
-                logger.lifecycle("Downloading $url")
-                url.url into file
-            }
-        }
-    }
-
     jar {
         val retained = configurations.compileClasspath.get() - exclusion.get()
         for (file in retained)

engines/ml/xgboost/src/main/java/ai/djl/ml/xgboost/XgbNDManager.java

@@ -109,8 +109,7 @@ public Engine getEngine() {
      * @return a new instance of {@link NDArray}
      */
     public NDArray create(ColumnBatch columnBatch, float missing, int nthread) {
-        columnBatch.getFeatureArrayInterface();
-        String json = columnBatch.getFeatureArrayInterface();
+        String json = columnBatch.toFeaturesJson();
         JsonArray array = JsonUtils.GSON.fromJson(json, JsonArray.class);
         JsonArray shapeJson = array.get(0).getAsJsonObject().get("shape").getAsJsonArray();
         long[] shapes = new long[shapeJson.size()];

engines/ml/xgboost/src/main/java/ml/dmlc/xgboost4j/java/JniUtils.java

@@ -56,7 +56,7 @@ public static long createDMatrix(Buffer buf, Shape shape, float missing) {
 
     public static long createDMatrix(ColumnBatch columnBatch, float missing, int nthread) {
         long[] handles = new long[1];
-        String json = columnBatch.getFeatureArrayInterface();
+        String json = columnBatch.toFeaturesJson();
         if (json == null || json.isEmpty()) {
             throw new IllegalArgumentException(
                     "Expecting non-empty feature columns' array interface");

engines/ml/xgboost/src/test/java/ai/djl/ml/xgboost/XgbModelTest.java

@@ -53,7 +53,7 @@ public void downloadXGBoostModel() throws IOException {
     @Test
     public void testVersion() {
         Engine engine = Engine.getEngine("XGBoost");
-        Assert.assertEquals("2.0.3", engine.getVersion());
+        Assert.assertEquals(engine.getVersion(), "3.0.4");
     }
 
     /*

gradle/libs.versions.toml

@@ -9,7 +9,7 @@ tensorflowCore = "1.0.0-rc.1"
 mxnet = "1.9.1"
 onnxruntime = "1.21.1"
 onnxruntimeExtensions = "0.13.0"
-xgboost = "2.0.3"
+xgboost = "3.0.4"
 lightgbm = "3.2.110"
 fasttext = "0.9.2"
 sentencepiece = "0.2.0"