diff --git a/CHANGELOG.md b/CHANGELOG.md index 404de2d6c..e38bef4fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ All notable changes to this project will be documented in this file. - hadoop: Add hdfs-utils ([#566]). - testing-tools: add pytest `8.0.1` ([#575]). - trino-cli: Command line for Trino version 440 ([#587]) +- spark: Add version `3.5.1` ([#588]). ### Changed @@ -107,6 +108,7 @@ All notable changes to this project will be documented in this file. [#579]: https://github.com/stackabletech/docker-images/pull/579 [#585]: https://github.com/stackabletech/docker-images/pull/585 [#587]: https://github.com/stackabletech/docker-images/pull/587 +[#588]: https://github.com/stackabletech/docker-images/pull/588 ## [23.11.0] - 2023-11-30 diff --git a/conf.py b/conf.py index 7d48a9611..576cfec1d 100644 --- a/conf.py +++ b/conf.py @@ -298,6 +298,23 @@ "jmx_exporter": "0.20.0", "tini": "0.19.0", }, + { + "product": "3.5.1", + "spark": "3.5.1", + "java-base": "11", + "python": "3.11", + "hadoop_short_version": "3", + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125 + "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 + "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.1 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 + "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 + "vector": "0.35.0", + "jmx_exporter": "0.20.0", + "tini": "0.19.0", + }, ], }, { diff --git a/druid/upload_new_druid_version.sh b/druid/upload_new_druid_version.sh index db2761e58..b04107e5d 100755 --- a/druid/upload_new_druid_version.sh +++ b/druid/upload_new_druid_version.sh @@ -44,7 +44,7 @@ if ! (sha512sum "${bin_file}" | cut -d " " -f 1 | diff -Z - "${bin_file}.sha512" fi echo "Validating signature" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://www.apache.org/dist/druid/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://www.apache.org/dist/druid/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/spark/KEYS | gpg --import")' if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then echo "ERROR: The signature could not be verified" diff --git a/hadoop/upload_new_hadoop_version.sh b/hadoop/upload_new_hadoop_version.sh index 3065bdd70..05b90d4b2 100755 --- a/hadoop/upload_new_hadoop_version.sh +++ b/hadoop/upload_new_hadoop_version.sh @@ -51,7 +51,7 @@ if ! (sha512sum --tag "$bin_file" | diff - "$bin_file.sha512" && sha512sum --tag fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/hadoop/common/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/hadoop/common/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/hadoop/common/KEYS | gpg --import")' if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" diff --git a/hbase/upload_new_hbase-operator-tools_version.sh b/hbase/upload_new_hbase-operator-tools_version.sh index 14ffdb62f..b0e7c5975 100755 --- a/hbase/upload_new_hbase-operator-tools_version.sh +++ b/hbase/upload_new_hbase-operator-tools_version.sh @@ -47,7 +47,7 @@ if ! (gpg --print-md SHA512 "$src_file" | diff - "$src_file.sha512"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/hbase/KEYS | gpg --import")' if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" exit 1 diff --git a/hbase/upload_new_hbase_version.sh b/hbase/upload_new_hbase_version.sh index 2a88dcc1d..3beaf25f5 100755 --- a/hbase/upload_new_hbase_version.sh +++ b/hbase/upload_new_hbase_version.sh @@ -53,7 +53,7 @@ if ! (gpg --print-md SHA512 "$bin_file" | diff - "$bin_file.sha512" && gpg --pri fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/hbase/KEYS | gpg --import")' if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" exit 1 diff --git a/hive/upload_new_hive_version.sh b/hive/upload_new_hive_version.sh index 98bdae51b..c592469be 100755 --- a/hive/upload_new_hive_version.sh +++ b/hive/upload_new_hive_version.sh @@ -44,7 +44,7 @@ if ! (sha256sum "${bin_file}" | diff - "${bin_file}.sha256"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://dlcdn.apache.org/hive/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://dlcdn.apache.org/hive/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://dlcdn.apache.org/hive/KEYS | gpg --import")' if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null); then echo "ERROR: Signature could not be verified" diff --git a/hive/upload_new_postgres_driver_version.sh b/hive/upload_new_postgres_driver_version.sh index 4d12bc789..b62dfad8d 100755 --- a/hive/upload_new_postgres_driver_version.sh +++ b/hive/upload_new_postgres_driver_version.sh @@ -55,7 +55,7 @@ if ! (sha1sum -c ${bin_file}.sha1 && sha1sum -c ${src_file}.sha1); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://raw.githubusercontent.com/pgjdbc/pgjdbc/master/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://raw.githubusercontent.com/pgjdbc/pgjdbc/master/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://raw.githubusercontent.com/pgjdbc/pgjdbc/master/KEYS | gpg --import")' if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null && gpg --verify "${src_file}.asc" "${src_file}" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" diff --git a/kafka/upload_new_kafka_version.sh b/kafka/upload_new_kafka_version.sh index ec79f924b..b3475ea98 100755 --- a/kafka/upload_new_kafka_version.sh +++ b/kafka/upload_new_kafka_version.sh @@ -44,7 +44,7 @@ if ! (gpg --print-md SHA512 "${bin_file}" | diff - "${bin_file}.sha512" ); then fi echo "Validating signature" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/kafka/KEYS | gpg --import")' if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then echo "ERROR: The signature could not be verified" diff --git a/nifi/upload_new_nifi_version.sh b/nifi/upload_new_nifi_version.sh index ca740f957..7b52099f7 100755 --- a/nifi/upload_new_nifi_version.sh +++ b/nifi/upload_new_nifi_version.sh @@ -45,7 +45,7 @@ if ! (sha512sum "$bin_file" | cut -d ' ' -f 1 | diff - "$bin_file.sha512"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/common/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/nifi/KEYS | gpg --import")' if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" diff --git a/spark-k8s/upload_new_spark_version.sh b/spark-k8s/upload_new_spark_version.sh index fc50a9462..5ced79905 100755 --- a/spark-k8s/upload_new_spark_version.sh +++ b/spark-k8s/upload_new_spark_version.sh @@ -44,7 +44,7 @@ if ! (sha512sum "${bin_file}" | diff - "${bin_file}.sha512"); then fi echo "Validating signature" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/spark/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/spark/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/spark/KEYS | gpg --import")' if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then echo "ERROR: The signature could not be verified" diff --git a/zookeeper/upload_new_zookeeper_version.sh b/zookeeper/upload_new_zookeeper_version.sh index ad08f3852..bda2d8486 100755 --- a/zookeeper/upload_new_zookeeper_version.sh +++ b/zookeeper/upload_new_zookeeper_version.sh @@ -45,7 +45,7 @@ if ! (sha512sum "$bin_file" | diff -Z - "$bin_file.sha512"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/zookeeper/KEYS) to GPG: https://www.apache.org/info/verification.html' +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/zookeeper/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/zookeeper/KEYS | gpg --import")' if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified"