From 009161ce61f8dcb9f97f6718bc0f6c5d0c696765 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 18 Oct 2023 08:27:01 +0900 Subject: [PATCH] Upgrade Delta version to 3.0.0 --- testing/spark3-delta/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/spark3-delta/Dockerfile b/testing/spark3-delta/Dockerfile index 4fadba16..d5e871d6 100644 --- a/testing/spark3-delta/Dockerfile +++ b/testing/spark3-delta/Dockerfile @@ -13,9 +13,9 @@ ARG ARCH FROM testing/centos7-oj11:unlabelled$ARCH -ARG SPARK_VERSION=3.4.0 +ARG SPARK_VERSION=3.5.0 ARG HADOOP_VERSION=3 -ARG DELTA_VERSION=2.4.0 +ARG DELTA_VERSION=3.0.0 ARG SCALA_VERSION=2.12 ARG SPARK_ARTIFACT="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" @@ -35,7 +35,7 @@ RUN wget -nv "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/ RUN wget -nv "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.319/aws-java-sdk-bundle-1.12.319.jar" # install Delta -RUN wget -nv "https://repo1.maven.org/maven2/io/delta/delta-core_${SCALA_VERSION}/${DELTA_VERSION}/delta-core_${SCALA_VERSION}-${DELTA_VERSION}.jar" +RUN wget -nv "https://repo1.maven.org/maven2/io/delta/delta-spark_${SCALA_VERSION}/${DELTA_VERSION}/delta-spark_${SCALA_VERSION}-${DELTA_VERSION}.jar" RUN wget -nv "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" # Create hive user to match Hive container