Add support for Spark 3.3.0-hadoop3.3

big-data-europe · Jul 1, 2022 · bc2a665 · bc2a665
1 parent 7050229
commit bc2a665
Show file tree

Hide file tree

Showing 21 changed files with 33 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -10,6 +10,7 @@ Docker images to:
 <details open>
 <summary>Currently supported versions:</summary>
 
+* Spark 3.3.0 for Hadoop 3.3 with OpenJDK 8 and Scala 2.12
 * Spark 3.2.1 for Hadoop 3.2 with OpenJDK 8 and Scala 2.12
 * Spark 3.2.0 for Hadoop 3.2 with OpenJDK 8 and Scala 2.12
 * Spark 3.1.2 for Hadoop 3.2 with OpenJDK 8 and Scala 2.12
@@ -52,15 +53,15 @@ Add the following services to your `docker-compose.yml` to integrate a Spark mas
 version: '3'
 services:
   spark-master:
-    image: bde2020/spark-master:3.2.1-hadoop3.2
+    image: bde2020/spark-master:3.3.0-hadoop3.3
     container_name: spark-master
     ports:
       - "8080:8080"
       - "7077:7077"
     environment:
       - INIT_DAEMON_STEP=setup_spark
   spark-worker-1:
-    image: bde2020/spark-worker:3.2.1-hadoop3.2
+    image: bde2020/spark-worker:3.3.0-hadoop3.3
     container_name: spark-worker-1
     depends_on:
       - spark-master
@@ -69,7 +70,7 @@ services:
     environment:
       - "SPARK_MASTER=spark://spark-master:7077"
   spark-worker-2:
-    image: bde2020/spark-worker:3.2.1-hadoop3.2
+    image: bde2020/spark-worker:3.3.0-hadoop3.3
     container_name: spark-worker-2
     depends_on:
       - spark-master
@@ -78,7 +79,7 @@ services:
     environment:
       - "SPARK_MASTER=spark://spark-master:7077"
   spark-history-server:
-      image: bde2020/spark-history-server:3.2.1-hadoop3.2
+      image: bde2020/spark-history-server:3.3.0-hadoop3.3
       container_name: spark-history-server
       depends_on:
         - spark-master
@@ -93,12 +94,12 @@ Make sure to fill in the `INIT_DAEMON_STEP` as configured in your pipeline.
 ### Spark Master
 To start a Spark master:
 
-    docker run --name spark-master -h spark-master -d bde2020/spark-master:3.2.1-hadoop3.2
+    docker run --name spark-master -h spark-master -d bde2020/spark-master:3.3.0-hadoop3.3
 
 ### Spark Worker
 To start a Spark worker:
 
-    docker run --name spark-worker-1 --link spark-master:spark-master -d bde2020/spark-worker:3.2.1-hadoop3.2
+    docker run --name spark-worker-1 --link spark-master:spark-master -d bde2020/spark-worker:3.3.0-hadoop3.3
 
 ## Launch a Spark application
 Building and running your Spark application on top of the Spark cluster is as simple as extending a template Docker image. Check the template's README for further documentation.
@@ -118,11 +119,11 @@ It will also setup a headless service so spark clients can be reachable from the
 
 Then to use `spark-shell` issue
 
-`kubectl run spark-base --rm -it --labels="app=spark-client" --image bde2020/spark-base:3.2.1-hadoop3.2 -- bash ./spark/bin/spark-shell --master spark://spark-master:7077 --conf spark.driver.host=spark-client`
+`kubectl run spark-base --rm -it --labels="app=spark-client" --image bde2020/spark-base:3.3.0-hadoop3.3 -- bash ./spark/bin/spark-shell --master spark://spark-master:7077 --conf spark.driver.host=spark-client`
 
 To use `spark-submit` issue for example
 
-`kubectl run spark-base --rm -it --labels="app=spark-client" --image bde2020/spark-base:3.2.1-hadoop3.2 -- bash ./spark/bin/spark-submit --class CLASS_TO_RUN --master spark://spark-master:7077 --deploy-mode client --conf spark.driver.host=spark-client URL_TO_YOUR_APP`
+`kubectl run spark-base --rm -it --labels="app=spark-client" --image bde2020/spark-base:3.3.0-hadoop3.3 -- bash ./spark/bin/spark-submit --class CLASS_TO_RUN --master spark://spark-master:7077 --deploy-mode client --conf spark.driver.host=spark-client URL_TO_YOUR_APP`
 
 You can use your own image packed with Spark and your application but when deployed it must be reachable from the workers.
 One way to achieve this is by creating a headless service for your pod and then use `--conf spark.driver.host=YOUR_HEADLESS_SERVICE` whenever you submit your application.
diff --git a/base/Dockerfile b/base/Dockerfile
@@ -7,8 +7,8 @@ ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon
 ENV INIT_DAEMON_STEP spark_master_init
 
 ENV BASE_URL=https://archive.apache.org/dist/spark/
-ENV SPARK_VERSION=3.2.1
-ENV HADOOP_VERSION=3.2
+ENV SPARK_VERSION=3.3.0
+ENV HADOOP_VERSION=3
 
 COPY wait-for-step.sh /
 COPY execute-step.sh /

diff --git a/build.sh b/build.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-TAG=3.2.1-hadoop3.2
+TAG=3.3.0-hadoop3.3
 
 build() {
     NAME=$1

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,15 +1,15 @@
 version: '3'
 services:
   spark-master:
-    image: bde2020/spark-master:3.2.1-hadoop3.2
+    image: bde2020/spark-master:3.3.0-hadoop3.3
     container_name: spark-master
     ports:
       - "8080:8080"
       - "7077:7077"
     environment:
       - INIT_DAEMON_STEP=setup_spark
   spark-worker-1:
-    image: bde2020/spark-worker:3.2.1-hadoop3.2
+    image: bde2020/spark-worker:3.3.0-hadoop3.3
     container_name: spark-worker-1
     depends_on:
       - spark-master

diff --git a/examples/maven/Dockerfile b/examples/maven/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-maven-template:3.2.1-hadoop3.2
+FROM bde2020/spark-maven-template:3.3.0-hadoop3.3
 
 LABEL MAINTAINER="Gezim Sejdiu <g.sejdiu@gmail.com>"
 

diff --git a/examples/maven/README.md b/examples/maven/README.md
@@ -17,5 +17,5 @@ To run the application, execute the following steps:
     ```
 3. Run the Docker container:
     ```bash
-    docker run --rm --network dockerspark_default --name spark-maven-example bde2020/spark-maven-example:3.2.1-hadoop3.2
+    docker run --rm --network dockerspark_default --name spark-maven-example bde2020/spark-maven-example:3.3.0-hadoop3.3
     ```
diff --git a/examples/maven/pom.xml b/examples/maven/pom.xml
@@ -14,7 +14,7 @@
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<scala.version>2.12.13</scala.version>
 		<scala.binary.version>2.12</scala.binary.version>
-		<spark.version>3.2.1</spark.version>
+		<spark.version>3.3.0</spark.version>
 	</properties>
 
 	<dependencies>

diff --git a/examples/python/Dockerfile b/examples/python/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-python-template:3.2.1-hadoop3.2
+FROM bde2020/spark-python-template:3.3.0-hadoop3.3
 
 COPY wordcount.py /app/
 

diff --git a/examples/python/README.md b/examples/python/README.md
@@ -17,5 +17,5 @@ To run the application, execute the following steps:
     ```
 3. Run the Docker container:
     ```bash
-    docker run --rm --network dockerspark_default --name pyspark-example bde2020/spark-python-example:3.2.1-hadoop3.2
+    docker run --rm --network dockerspark_default --name pyspark-example bde2020/spark-python-example:3.3.0-hadoop3.3
     ```
diff --git a/history-server/Dockerfile b/history-server/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-base:3.2.1-hadoop3.2
+FROM bde2020/spark-base:3.3.0-hadoop3.3
 
 LABEL maintainer="Gezim Sejdiu <g.sejdiu@gmail.com>, Giannis Mouchakis <gmouchakis@gmail.com>"
 

diff --git a/k8s-spark-cluster.yaml b/k8s-spark-cluster.yaml
@@ -46,7 +46,7 @@ spec:
     spec:
       containers:
       - name: spark-master
-        image: bde2020/spark-master:3.2.1-hadoop3.2
+        image: bde2020/spark-master:3.3.0-hadoop3.3
         imagePullPolicy: Always
         ports:
         - containerPort: 8080
@@ -70,7 +70,7 @@ spec:
     spec:
       containers:
       - name: spark-worker
-        image: bde2020/spark-worker:3.2.1-hadoop3.2
+        image: bde2020/spark-worker:3.3.0-hadoop3.3
         imagePullPolicy: Always
         ports:
         - containerPort: 8081
diff --git a/master/Dockerfile b/master/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-base:3.2.1-hadoop3.2
+FROM bde2020/spark-base:3.3.0-hadoop3.3
 
 LABEL maintainer="Gezim Sejdiu <g.sejdiu@gmail.com>, Giannis Mouchakis <gmouchakis@gmail.com>"
 

diff --git a/submit/Dockerfile b/submit/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-base:3.2.1-hadoop3.2
+FROM bde2020/spark-base:3.3.0-hadoop3.3
 
 LABEL maintainer="Gezim Sejdiu <g.sejdiu@gmail.com>, Giannis Mouchakis <gmouchakis@gmail.com>"
 

diff --git a/template/maven/Dockerfile b/template/maven/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-submit:3.2.1-hadoop3.2
+FROM bde2020/spark-submit:3.3.0-hadoop3.3
 
 LABEL maintainer="Gezim Sejdiu <g.sejdiu@gmail.com>, Giannis Mouchakis <gmouchakis@gmail.com>"
 

diff --git a/template/maven/README.md b/template/maven/README.md
@@ -34,7 +34,7 @@ If you overwrite the template's `CMD` in your Dockerfile, make sure to execute t
 
 #### Example Dockerfile
 ```
-FROM bde2020/spark-maven-template:3.2.1-hadoop3.2
+FROM bde2020/spark-maven-template:3.3.0-hadoop3.3
 
 MAINTAINER Erika Pauwels <erika.pauwels@tenforce.com>
 MAINTAINER Gezim Sejdiu <g.sejdiu@gmail.com>

diff --git a/template/python/Dockerfile b/template/python/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-submit:3.2.1-hadoop3.2
+FROM bde2020/spark-submit:3.3.0-hadoop3.3
 
 LABEL maintainer="Gezim Sejdiu <g.sejdiu@gmail.com>, Giannis Mouchakis <gmouchakis@gmail.com>"
 

diff --git a/template/python/README.md b/template/python/README.md
@@ -30,7 +30,7 @@ If you overwrite the template's `CMD` in your Dockerfile, make sure to execute t
 
 #### Example Dockerfile
 ```
-FROM bde2020/spark-python-template:3.2.1-hadoop3.2
+FROM bde2020/spark-python-template:3.3.0-hadoop3.3
 
 MAINTAINER You <you@example.org>
 

diff --git a/template/sbt/Dockerfile b/template/sbt/Dockerfile
@@ -1,9 +1,9 @@
-FROM bde2020/spark-submit:3.2.1-hadoop3.2
+FROM bde2020/spark-submit:3.3.0-hadoop3.3
 
 LABEL maintainer="Gezim Sejdiu <g.sejdiu@gmail.com>, Giannis Mouchakis <gmouchakis@gmail.com>"
 
 ARG SBT_VERSION
-ENV SBT_VERSION=${SBT_VERSION:-1.4.1}
+ENV SBT_VERSION=${SBT_VERSION:-1.6.2}
 
 RUN wget -O - https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz | gunzip | tar -x -C /usr/local
 

diff --git a/template/sbt/README.md b/template/sbt/README.md
@@ -62,7 +62,7 @@ the `/template.sh` script at the end.
 #### Example Dockerfile
 
 ```
-FROM bde2020/spark-sbt-template:3.2.1-hadoop3.2
+FROM bde2020/spark-sbt-template:3.3.0-hadoop3.3
 
 MAINTAINER Cecile Tonglet <cecile.tonglet@tenforce.com>
 

diff --git a/template/sbt/build.sbt b/template/sbt/build.sbt
@@ -1,4 +1,4 @@
 scalaVersion := "2.12.14"
 libraryDependencies ++= Seq(
-  "org.apache.spark" %% "spark-sql" % "3.2.1" % "provided"
+  "org.apache.spark" %% "spark-sql" % "3.3.0" % "provided"
 )
diff --git a/worker/Dockerfile b/worker/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/spark-base:3.2.1-hadoop3.2
+FROM bde2020/spark-base:3.3.0-hadoop3.3
 
 LABEL maintainer="Gezim Sejdiu <g.sejdiu@gmail.com>, Giannis Mouchakis <gmouchakis@gmail.com>"