stackabletech · maltesander · Oct 17, 2023 · Oct 10, 2023 · Oct 10, 2023 · Oct 10, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.
 - nifi: added support for version 1.23.2 ([#473]).
 - opa: add version 0.57.0 ([#471]).
 - opa: add new version upload script ([#471]).
+- spark: added versions 3.4.1, 3.5.0 ([#475]).
 - superset: add new version 2.1.1, 3.0.0 ([#482]).
 - zookeeper: add version 3.8.3 ([#470]).
 - zookeeper: add upload script ([#470]).
@@ -38,6 +39,7 @@ All notable changes to this project will be documented in this file.
 - airflow: Added wrapper script to allow the triggering of pre/post hook actions ([#435]).
 - hadoop: bumped jmx-exporter version to 0.20.0 ([#478]).
 - hive: bump jmx-exporter to 0.20.0 ([#472]).
+- spark: bump jmx-exporter to 0.20.0 and access via softlink ([#475]).
 - superset: removed patches that are obsolete since 2.0.0 ([#482]).
 - superset: bump statsd_exporter to 0.24.0 and set via conf.py ([#482]).
 - zookeeper: bumped jmx-exporter version to 0.20.0 ([#470]).
@@ -55,6 +57,7 @@ All notable changes to this project will be documented in this file.
 - nifi: removed support for version 1.15.x, 1.16.x, 1.18.x, 1.20.x ([#473]).
 - nifi: removed openssl from image ([#473]).
 - opa: removed versions 0.27.1, 0.28.0, 0.37.2, 0.41.0, 0.45.0 ([#471]).
+- spark: removed versions 3.2.1, 3.3.0 versions ([#475]).
 - superset: removed versions 1.3.2, 1.4.1, 1.4.2, 1.5.1, 1.5.3, 2.0.1 ([#482]).
 - zookeeper: removed versions 3.5.8, 3.6.3, 3.7.0, 3.8.0 ([#470]).
 
@@ -77,6 +80,7 @@ All notable changes to this project will be documented in this file.
 [#471]: https://github.com/stackabletech/docker-images/pull/471
 [#472]: https://github.com/stackabletech/docker-images/pull/472
 [#473]: https://github.com/stackabletech/docker-images/pull/473
+[#475]: https://github.com/stackabletech/docker-images/pull/475
 [#476]: https://github.com/stackabletech/docker-images/pull/476
 [#478]: https://github.com/stackabletech/docker-images/pull/478
 [#479]: https://github.com/stackabletech/docker-images/pull/479

diff --git a/conf.py b/conf.py
@@ -219,51 +219,6 @@
     {
         "name": "spark-k8s",
         "versions": [
-            {
-                "product": "3.2.1",
-                "spark": "3.2.1",
-                "java-base": "11",
-                "python": "39",
-                "hadoop_short_version": "3.2",
-                "hadoop_long_version": "3.3.1",
-                "aws_java_sdk_bundle": "1.11.901",
-                "azure_storage": "7.0.1",
-                "azure_keyvault_core": "1.0.0",
-                "jackson_dataformat_xml": "2.12.3",
-                "stax2_api": "4.2.1",
-                "woodstox_core": "6.2.1",
-                "vector": "0.33.0",
-            },
-            {
-                "product": "3.3.0",
-                "spark": "3.3.0",
-                "java-base": "11",
-                "python": "39",
-                "hadoop_short_version": "3",
-                "hadoop_long_version": "3.3.3",
-                "aws_java_sdk_bundle": "1.11.1026",
-                "azure_storage": "7.0.1",
-                "azure_keyvault_core": "1.0.0",
-                "jackson_dataformat_xml": "2.13.3",
-                "stax2_api": "4.2.1",
-                "woodstox_core": "6.2.1",
-                "vector": "0.33.0",
-            },
-            {
-                "product": "3.3.0-java17",
-                "spark": "3.3.0",
-                "java-base": "17",
-                "python": "39",
-                "hadoop_short_version": "3",
-                "hadoop_long_version": "3.3.3",
-                "aws_java_sdk_bundle": "1.11.1026",
-                "azure_storage": "7.0.1",
-                "azure_keyvault_core": "1.0.0",
-                "jackson_dataformat_xml": "2.13.3",
-                "stax2_api": "4.2.1",
-                "woodstox_core": "6.2.1",
-                "vector": "0.33.0",
-            },
             {
                 "product": "3.4.0",
                 "spark": "3.4.0",
@@ -278,7 +233,9 @@
                 "stax2_api": "4.2.1",
                 "woodstox_core": "6.5.0",
                 "vector": "0.33.0",
+                "jmx_exporter": "0.20.0",
             },
+            # required for a customer
             {
                 "product": "3.4.0-java17",
                 "spark": "3.4.0",
@@ -293,6 +250,39 @@
                 "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
                 "woodstox_core": "6.5.0",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
                 "vector": "0.33.0",
+                "jmx_exporter": "0.20.0",
+            },
+            {
+                "product": "3.4.1",
+                "spark": "3.4.1",
+                "java-base": "11",
+                "python": "3.11",
+                "hadoop_short_version": "3",
+                "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122
+                "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
+                "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
+                "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+                "jackson_dataformat_xml": "2.14.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0
+                "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
+                "woodstox_core": "6.5.0",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
+                "vector": "0.33.0",
+                "jmx_exporter": "0.20.0",
+            },
+            {
+                "product": "3.5.0",
+                "spark": "3.5.0",
+                "java-base": "11",
+                "python": "3.11",
+                "hadoop_short_version": "3",
+                "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125
+                "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
+                "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
+                "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+                "jackson_dataformat_xml": "2.15.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.0
+                "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+                "woodstox_core": "6.5.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+                "vector": "0.33.0",
+                "jmx_exporter": "0.20.0",
             },
         ],
     },

diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile
@@ -12,6 +12,7 @@ ARG AZURE_KEYVAULT_CORE
 ARG JACKSON_DATAFORMAT_XML
 ARG STAX2_API
 ARG WOODSTOX_CORE
+ARG JMX_EXPORTER
 ARG RELEASE
 
 LABEL name="Apache Spark" \
@@ -91,10 +92,9 @@ RUN mkdir /stackable/spark/extra-jars && \
     curl -L --fail -o /stackable/spark/extra-jars/woodstox-core-${WOODSTOX_CORE}.jar \
     https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar
 
-# Download jmx exporter (needed for metrics)
-RUN curl -L --fail https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-0.16.1.jar \
-    -o /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar && \
-    chmod -x /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar
+RUN curl --fail "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \
+    chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \
+    ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar
 
 # Symlink example jar, so that we can easily use it in tests
 RUN ln -s /stackable/spark/examples/jars/spark-examples_*.jar /stackable/spark/examples/jars/spark-examples.jar

diff --git a/spark-k8s/upload_new_spark_version.sh b/spark-k8s/upload_new_spark_version.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+VERSION=${1:?"Missing version number argument (arg 1)"}
+NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"}
+
+read -r -s -p "Nexus Password: " NEXUS_PASSWORD
+echo ""
+
+# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory
+# Find the directory name of the script
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# the temp directory used, within $DIR
+WORK_DIR=$(mktemp -d -p "$DIR")
+
+# check if tmp dir was created
+if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
+  echo "Could not create temp dir"
+  exit 1
+fi
+
+# deletes the temp directory
+function cleanup {
+  rm -rf "$WORK_DIR"
+}
+
+# register the cleanup function to be called on the EXIT signal
+trap cleanup EXIT
+
+cd "$WORK_DIR" || exit
+
+bin_file="spark-${VERSION}-bin-hadoop3.tgz"
+
+echo "Downloading Spark (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)"
+curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${bin_file}"
+curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${bin_file}.asc"
+curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${bin_file}.sha512"
+
+# It is probably redundant to check both the checksum and the signature but it's cheap and why not
+echo "Validating SHA512 Checksum"
+if ! (sha512sum "${bin_file}" | diff - "${bin_file}.sha512"); then
+  echo "ERROR: The SHA512 sum does not match"
+  exit 1
+fi
+
+echo "Validating signature"
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/spark/KEYS) to GPG: https://www.apache.org/info/verification.html'
+
+if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then
+  echo "ERROR: The signature could not be verified"
+  exit 1
+fi
+
+echo "Uploading everything to Nexus"
+EXIT_STATUS=0
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$?
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$?
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$?
+
+if [ $EXIT_STATUS -ne 0 ]; then
+  echo "ERROR: Upload failed"
+  exit 1
+fi
+
+echo "Successfully uploaded version ${VERSION} of Spark to Nexus"
+echo "https://repo.stackable.tech/service/rest/repository/browse/packages/spark/"