From 50da1b0af4d1f6588b6083eee50c5a5acea276bc Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Thu, 28 Mar 2024 21:37:06 +0800 Subject: [PATCH] fix --- .github/workflows/gluten.yml | 5 ++--- integration-tests/kyuubi-gluten-it/pom.xml | 8 ++++++++ .../test/scala/org/apache/kyuubi/GlutenSuiteMixin.scala | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gluten.yml b/.github/workflows/gluten.yml index 7ac5c19176e..6c87d04f8c3 100644 --- a/.github/workflows/gluten.yml +++ b/.github/workflows/gluten.yml @@ -64,10 +64,9 @@ jobs: if [[ "${{ steps.gluten-cache.outputs.cache-hit }}" != 'true' ]]; then git clone https://github.com/oap-project/gluten.git cd gluten - ./dev/builddeps-veloxbe.sh + ./dev/package.sh rm -rf dist && mkdir -p dist - mvn clean package -Pbackends-velox -Prss -Pspark-3.2 -DskipTests - cp package/target/gluten-velox-bundle-spark*.jar dist + cp package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar dist mvn clean package -Pbackends-velox -Prss -Pspark-3.3 -DskipTests cp package/target/gluten-velox-bundle-spark*.jar dist mvn clean package -Pbackends-velox -Prss -Pspark-3.4 -DskipTests diff --git a/integration-tests/kyuubi-gluten-it/pom.xml b/integration-tests/kyuubi-gluten-it/pom.xml index b82a272b287..9a6ccb84ba9 100644 --- a/integration-tests/kyuubi-gluten-it/pom.xml +++ b/integration-tests/kyuubi-gluten-it/pom.xml @@ -43,6 +43,14 @@ ${project.basedir}/../../gluten/dist/${gluten.artifact.name}_x86_64-${gluten.version}.jar + + io.glutenproject + gluten-thirdparty-lib + ${gluten.version} + system + ${project.basedir}/../../gluten/dist/gluten-thirdparty-lib-ubuntu-22.04-x86_64.jar + + org.apache.kyuubi kyuubi-common_${scala.binary.version} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/GlutenSuiteMixin.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/GlutenSuiteMixin.scala index c9ee6ab9625..e9d372b5d78 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/GlutenSuiteMixin.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/GlutenSuiteMixin.scala @@ -21,7 +21,7 @@ trait GlutenSuiteMixin { protected def extraJars: String = { System.getProperty("java.class.path") .split(":") - .filter(_.contains("gluten-velox-bundle-spark")).head + .filter(_.contains("gluten-")).mkString(",") } protected def extraConfigs: Map[String, String] = Map( @@ -30,5 +30,6 @@ trait GlutenSuiteMixin { "spark.memory.offHeap.enabled" -> "true", "spark.shuffle.manager" -> "org.apache.spark.shuffle.sort.ColumnarShuffleManager", "spark.gluten.ui.enabled" -> "false", + "spark.gluten.loadLibFromJar" -> "true", "spark.jars" -> extraJars) }