From 3855d8dd5af9fde55b72bdff263d72f7e17a139c Mon Sep 17 00:00:00 2001 From: "Qian.Sun" Date: Wed, 23 Feb 2022 18:18:37 +0800 Subject: [PATCH 1/2] [SPARK-38302][K8S][TESTS] Use Java 17 in K8S integration tests when setting spark-tgz --- project/SparkBuild.scala | 2 +- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- .../integration-tests/scripts/setup-integration-test-env.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index e9ef514c4e331..0f06e6bcb0897 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -645,7 +645,7 @@ object KubernetesIntegrationTests { val bindingsDir = s"$sparkHome/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings" val javaImageTag = sys.props.get("spark.kubernetes.test.javaImageTag") val dockerFile = sys.props.getOrElse("spark.kubernetes.test.dockerFile", - "resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile.java17") + s"$sparkHome/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile.java17") val extraOptions = if (javaImageTag.isDefined) { Seq("-b", s"java_image_tag=$javaImageTag") } else { diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 0bc8508cbf86c..8e23d96579041 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -43,7 +43,7 @@ - resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile.java17 + /Dockerfile.java17 diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh index e4a92b60c981d..870e5d8a6b4ce 100755 --- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -106,7 +106,7 @@ then # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim) JAVA_IMAGE_TAG_BUILD_ARG="-b java_image_tag=$JAVA_IMAGE_TAG" else - JAVA_IMAGE_TAG_BUILD_ARG="-f $DOCKER_FILE" + JAVA_IMAGE_TAG_BUILD_ARG="-f $DOCKER_FILE_BASE_PATH$DOCKER_FILE" fi # Build PySpark image From 2e85e70637ab77fc7fd3da596958e70880a11e29 Mon Sep 17 00:00:00 2001 From: "Qian.Sun" Date: Fri, 25 Feb 2022 23:18:38 +0800 Subject: [PATCH 2/2] allow docker-file to be absolute path --- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- .../integration-tests/scripts/setup-integration-test-env.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 8e23d96579041..318a903c14215 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -43,7 +43,7 @@ - /Dockerfile.java17 + Dockerfile.java17 diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh index 870e5d8a6b4ce..d8960349f0080 100755 --- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -106,7 +106,11 @@ then # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim) JAVA_IMAGE_TAG_BUILD_ARG="-b java_image_tag=$JAVA_IMAGE_TAG" else - JAVA_IMAGE_TAG_BUILD_ARG="-f $DOCKER_FILE_BASE_PATH$DOCKER_FILE" + if [[ $DOCKER_FILE = /* ]]; then + JAVA_IMAGE_TAG_BUILD_ARG="-f $DOCKER_FILE" + else + JAVA_IMAGE_TAG_BUILD_ARG="-f $DOCKER_FILE_BASE_PATH/$DOCKER_FILE" + fi fi # Build PySpark image