From 5299481da2f38f3678a159406a9fb9f37c42529e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 12:18:48 +0200 Subject: [PATCH 1/7] add spark upload script --- spark-k8s/upload_new_spark_version.sh | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 spark-k8s/upload_new_spark_version.sh diff --git a/spark-k8s/upload_new_spark_version.sh b/spark-k8s/upload_new_spark_version.sh new file mode 100755 index 000000000..fc50a9462 --- /dev/null +++ b/spark-k8s/upload_new_spark_version.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +VERSION=${1:?"Missing version number argument (arg 1)"} +NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} + +read -r -s -p "Nexus Password: " NEXUS_PASSWORD +echo "" + +# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory +# Find the directory name of the script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# the temp directory used, within $DIR +WORK_DIR=$(mktemp -d -p "$DIR") + +# check if tmp dir was created +if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then + echo "Could not create temp dir" + exit 1 +fi + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +cd "$WORK_DIR" || exit + +bin_file="spark-${VERSION}-bin-hadoop3.tgz" + +echo "Downloading Spark (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" +curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${bin_file}" +curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${bin_file}.asc" +curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${bin_file}.sha512" + +# It is probably redundant to check both the checksum and the signature but it's cheap and why not +echo "Validating SHA512 Checksum" +if ! (sha512sum "${bin_file}" | diff - "${bin_file}.sha512"); then + echo "ERROR: The SHA512 sum does not match" + exit 1 +fi + +echo "Validating signature" +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/spark/KEYS) to GPG: https://www.apache.org/info/verification.html' + +if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then + echo "ERROR: The signature could not be verified" + exit 1 +fi + +echo "Uploading everything to Nexus" +EXIT_STATUS=0 +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? + +if [ $EXIT_STATUS -ne 0 ]; then + echo "ERROR: Upload failed" + exit 1 +fi + +echo "Successfully uploaded version ${VERSION} of Spark to Nexus" +echo "https://repo.stackable.tech/service/rest/repository/browse/packages/spark/" From 21150dd4f131699abf0c07ae1c68b38d94667f94 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 12:18:59 +0200 Subject: [PATCH 2/7] add new versions, remove old and java 17 versions --- conf.py | 69 ++++++++++++++++++--------------------------------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/conf.py b/conf.py index 8aa3ae2b4..c333c02bb 100644 --- a/conf.py +++ b/conf.py @@ -252,51 +252,6 @@ { "name": "spark-k8s", "versions": [ - { - "product": "3.2.1", - "spark": "3.2.1", - "java-base": "11", - "python": "39", - "hadoop_short_version": "3.2", - "hadoop_long_version": "3.3.1", - "aws_java_sdk_bundle": "1.11.901", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.12.3", - "stax2_api": "4.2.1", - "woodstox_core": "6.2.1", - "vector": "0.31.0", - }, - { - "product": "3.3.0", - "spark": "3.3.0", - "java-base": "11", - "python": "39", - "hadoop_short_version": "3", - "hadoop_long_version": "3.3.3", - "aws_java_sdk_bundle": "1.11.1026", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.13.3", - "stax2_api": "4.2.1", - "woodstox_core": "6.2.1", - "vector": "0.31.0", - }, - { - "product": "3.3.0-java17", - "spark": "3.3.0", - "java-base": "17", - "python": "39", - "hadoop_short_version": "3", - "hadoop_long_version": "3.3.3", - "aws_java_sdk_bundle": "1.11.1026", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.13.3", - "stax2_api": "4.2.1", - "woodstox_core": "6.2.1", - "vector": "0.31.0", - }, { "product": "3.4.0", "spark": "3.4.0", @@ -311,11 +266,28 @@ "stax2_api": "4.2.1", "woodstox_core": "6.5.0", "vector": "0.31.0", + "jmx_exporter": "0.20.0", }, { - "product": "3.4.0-java17", - "spark": "3.4.0", - "java-base": "17", + "product": "3.4.1", + "spark": "3.4.1", + "java-base": "11", + "python": "3.11", + "hadoop_short_version": "3", + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 + "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "vector": "0.31.0", + "jmx_exporter": "0.20.0", + }, + { + "product": "3.5.0", + "spark": "3.5.0", + "java-base": "11", "python": "3.11", "hadoop_short_version": "3", "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 @@ -326,6 +298,7 @@ "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "vector": "0.31.0", + "jmx_exporter": "0.20.0", }, ], }, From 0aecea1118c248ee47c8c036cc48ddf9d76cee2b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 13:06:22 +0200 Subject: [PATCH 3/7] adapt versions --- conf.py | 64 ++++++++++++++++++++++---------------------- spark-k8s/Dockerfile | 8 +++--- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/conf.py b/conf.py index c333c02bb..5004cb0ee 100644 --- a/conf.py +++ b/conf.py @@ -252,22 +252,22 @@ { "name": "spark-k8s", "versions": [ - { - "product": "3.4.0", - "spark": "3.4.0", - "java-base": "11", - "python": "3.11", - "hadoop_short_version": "3", - "hadoop_long_version": "3.3.4", - "aws_java_sdk_bundle": "1.12.262", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.14.2", - "stax2_api": "4.2.1", - "woodstox_core": "6.5.0", - "vector": "0.31.0", - "jmx_exporter": "0.20.0", - }, + # { + # "product": "3.4.0", + # "spark": "3.4.0", + # "java-base": "11", + # "python": "3.11", + # "hadoop_short_version": "3", + # "hadoop_long_version": "3.3.4", + # "aws_java_sdk_bundle": "1.12.262", + # "azure_storage": "7.0.1", + # "azure_keyvault_core": "1.0.0", + # "jackson_dataformat_xml": "2.14.2", + # "stax2_api": "4.2.1", + # "woodstox_core": "6.5.0", + # "vector": "0.31.0", + # "jmx_exporter": "0.20.0", + # }, { "product": "3.4.1", "spark": "3.4.1", @@ -284,22 +284,22 @@ "vector": "0.31.0", "jmx_exporter": "0.20.0", }, - { - "product": "3.5.0", - "spark": "3.5.0", - "java-base": "11", - "python": "3.11", - "hadoop_short_version": "3", - "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 - "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 - "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 - "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 - "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "vector": "0.31.0", - "jmx_exporter": "0.20.0", - }, + # { + # "product": "3.5.0", + # "spark": "3.5.0", + # "java-base": "11", + # "python": "3.11", + # "hadoop_short_version": "3", + # "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + # "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 + # "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 + # "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + # "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 + # "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + # "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + # "vector": "0.31.0", + # "jmx_exporter": "0.20.0", + # }, ], }, { diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index bac4a1e85..82732d641 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -12,6 +12,7 @@ ARG AZURE_KEYVAULT_CORE ARG JACKSON_DATAFORMAT_XML ARG STAX2_API ARG WOODSTOX_CORE +ARG JMX_EXPORTER ARG RELEASE LABEL name="Apache Spark" \ @@ -91,10 +92,9 @@ RUN mkdir /stackable/spark/extra-jars && \ curl -L --fail -o /stackable/spark/extra-jars/woodstox-core-${WOODSTOX_CORE}.jar \ https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar -# Download jmx exporter (needed for metrics) -RUN curl -L --fail https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-0.16.1.jar \ - -o /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar && \ - chmod -x /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar +RUN curl --fail "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \ + chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \ + ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar # Symlink example jar, so that we can easily use it in tests RUN ln -s /stackable/spark/examples/jars/spark-examples_*.jar /stackable/spark/examples/jars/spark-examples.jar From c135d88532c2e7bb12ab77891a3a7feb97d4b030 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 13:24:59 +0200 Subject: [PATCH 4/7] keep java 17 3.4.0 image --- conf.py | 80 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/conf.py b/conf.py index 5004cb0ee..584bd9ec9 100644 --- a/conf.py +++ b/conf.py @@ -252,22 +252,38 @@ { "name": "spark-k8s", "versions": [ - # { - # "product": "3.4.0", - # "spark": "3.4.0", - # "java-base": "11", - # "python": "3.11", - # "hadoop_short_version": "3", - # "hadoop_long_version": "3.3.4", - # "aws_java_sdk_bundle": "1.12.262", - # "azure_storage": "7.0.1", - # "azure_keyvault_core": "1.0.0", - # "jackson_dataformat_xml": "2.14.2", - # "stax2_api": "4.2.1", - # "woodstox_core": "6.5.0", - # "vector": "0.31.0", - # "jmx_exporter": "0.20.0", - # }, + { + "product": "3.4.0", + "spark": "3.4.0", + "java-base": "11", + "python": "3.11", + "hadoop_short_version": "3", + "hadoop_long_version": "3.3.4", + "aws_java_sdk_bundle": "1.12.262", + "azure_storage": "7.0.1", + "azure_keyvault_core": "1.0.0", + "jackson_dataformat_xml": "2.14.2", + "stax2_api": "4.2.1", + "woodstox_core": "6.5.0", + "vector": "0.31.0", + "jmx_exporter": "0.20.0", + }, + # required for a customer + { + "product": "3.4.0-java17", + "spark": "3.4.0", + "java-base": "17", + "python": "3.11", + "hadoop_short_version": "3", + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 + "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "vector": "0.31.0", + }, { "product": "3.4.1", "spark": "3.4.1", @@ -284,22 +300,22 @@ "vector": "0.31.0", "jmx_exporter": "0.20.0", }, - # { - # "product": "3.5.0", - # "spark": "3.5.0", - # "java-base": "11", - # "python": "3.11", - # "hadoop_short_version": "3", - # "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 - # "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 - # "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 - # "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - # "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 - # "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - # "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - # "vector": "0.31.0", - # "jmx_exporter": "0.20.0", - # }, + { + "product": "3.5.0", + "spark": "3.5.0", + "java-base": "11", + "python": "3.11", + "hadoop_short_version": "3", + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 + "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "vector": "0.31.0", + "jmx_exporter": "0.20.0", + }, ], }, { From a02b8c1235caefdd17cf9b5e28959c7e43325cd1 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 13:26:49 +0200 Subject: [PATCH 5/7] adapted changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64550625c..1a8b80de3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file. - java-base: Add needed tzdata-java package ([#425]). - testing-tools: Add java, tzdata-java, unzip ([#464], [#465], [#466]). +- spark: added versions 3.4.1, 3.5.0 ([#475]). + ### Changed - Extract image tools their own [repository](https://github.com/stackabletech/image-tools) ([#437]) @@ -27,6 +29,8 @@ All notable changes to this project will be documented in this file. - airflow: Updated statsd-exporter to 0.24, this was accidentally moved to a very old version previously (0.3.0) ([#431]). - airflow: Added wrapper script to allow the triggering of pre/post hook actions ([#435]). +- spark: bump jmx-exporter to 0.20.0 and access via softlink ([#475]). + ### Removed - airflow: Remove unused environment variable `AIRFLOW_UID` ([#429]). @@ -34,6 +38,8 @@ All notable changes to this project will be documented in this file. - pyspark-k8s: The PySpark image has been removed completely. Python is now installed with the Spark image ([#436]) - Removed all product specific changelogs and updated the root file ([#440]) +- spark: removed versions 3.2.1, 3.3.0 versions ([#475]). + [#400]: https://github.com/stackabletech/docker-images/pull/400 [#419]: https://github.com/stackabletech/docker-images/pull/419 [#425]: https://github.com/stackabletech/docker-images/pull/425 @@ -49,6 +55,7 @@ All notable changes to this project will be documented in this file. [#464]: https://github.com/stackabletech/docker-images/pull/464 [#465]: https://github.com/stackabletech/docker-images/pull/465 [#466]: https://github.com/stackabletech/docker-images/pull/466 +[#475]: https://github.com/stackabletech/docker-images/pull/475 ## [23.7.0] - 2023-07-14 From 08a03beafbd52381eb416863d9bb04990d9bc6db Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 17 Oct 2023 17:14:17 +0200 Subject: [PATCH 6/7] added missing jmx exporter --- conf.py | 78 +++++++++++++++++++++++++-------------------------------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/conf.py b/conf.py index 6343387dc..9263078c7 100644 --- a/conf.py +++ b/conf.py @@ -220,69 +220,58 @@ "name": "spark-k8s", "versions": [ { - "product": "3.2.1", - "spark": "3.2.1", - "java-base": "11", - "python": "39", - "hadoop_short_version": "3.2", - "hadoop_long_version": "3.3.1", - "aws_java_sdk_bundle": "1.11.901", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.12.3", - "stax2_api": "4.2.1", - "woodstox_core": "6.2.1", - "vector": "0.33.0", - }, - { - "product": "3.3.0", - "spark": "3.3.0", + "product": "3.4.0", + "spark": "3.4.0", "java-base": "11", - "python": "39", + "python": "3.11", "hadoop_short_version": "3", - "hadoop_long_version": "3.3.3", - "aws_java_sdk_bundle": "1.11.1026", + "hadoop_long_version": "3.3.4", + "aws_java_sdk_bundle": "1.12.262", "azure_storage": "7.0.1", "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.13.3", + "jackson_dataformat_xml": "2.14.2", "stax2_api": "4.2.1", - "woodstox_core": "6.2.1", + "woodstox_core": "6.5.0", "vector": "0.33.0", + "jmx_exporter": "0.20.0", }, + # required for a customer { - "product": "3.3.0-java17", - "spark": "3.3.0", + "product": "3.4.0-java17", + "spark": "3.4.0", "java-base": "17", - "python": "39", + "python": "3.11", "hadoop_short_version": "3", - "hadoop_long_version": "3.3.3", - "aws_java_sdk_bundle": "1.11.1026", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.13.3", - "stax2_api": "4.2.1", - "woodstox_core": "6.2.1", + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 + "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "vector": "0.33.0", + "jmx_exporter": "0.20.0", }, { - "product": "3.4.0", - "spark": "3.4.0", + "product": "3.4.1", + "spark": "3.4.1", "java-base": "11", "python": "3.11", "hadoop_short_version": "3", - "hadoop_long_version": "3.3.4", - "aws_java_sdk_bundle": "1.12.262", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - "jackson_dataformat_xml": "2.14.2", - "stax2_api": "4.2.1", - "woodstox_core": "6.5.0", + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 + "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "vector": "0.33.0", + "jmx_exporter": "0.20.0", }, { - "product": "3.4.0-java17", - "spark": "3.4.0", - "java-base": "17", + "product": "3.5.0", + "spark": "3.5.0", + "java-base": "11", "python": "3.11", "hadoop_short_version": "3", "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 @@ -293,6 +282,7 @@ "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 "vector": "0.33.0", + "jmx_exporter": "0.20.0", }, ], }, From f4ae7a72bab831f0a786fa2f69f3168cf1e4a2b3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 17 Oct 2023 17:35:31 +0200 Subject: [PATCH 7/7] fix sources of dependencies --- conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf.py b/conf.py index 9263078c7..61277d0f3 100644 --- a/conf.py +++ b/conf.py @@ -274,13 +274,13 @@ "java-base": "11", "python": "3.11", "hadoop_short_version": "3", - "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125 "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.0 - "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.0 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 + "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 "vector": "0.33.0", "jmx_exporter": "0.20.0", },