From 0e967bfe2fa96e5a758d54c562ab3b93ee0eb710 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jul 2024 11:56:28 +0200 Subject: [PATCH 1/3] add spark version 4.0.0-preview1 --- docs/modules/spark-k8s/partials/supported-versions.adoc | 8 ++++---- tests/test-definition.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index cd1a86c2..0a6fd493 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,7 +3,7 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) -- 3.5.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) -- 3.4.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (LTS) -- 3.4.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) +- 4.0.0-preview1 (Hadoop 3.4.0, Scala 2.12, Python 3.11, Java 17) (experimental) +- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) +- 3.4.3 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) +- 3.4.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 0ab33e6d..2b3bc9c1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,10 +5,10 @@ dimensions: - "false" - name: spark values: - - 3.4.1 - 3.4.2 - - 3.5.0 + - 3.4.3 - 3.5.1 + - 4.0.0-preview1 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev From 41ee49ffc66eb647ce8b409851b577c0bf486f5d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jul 2024 14:38:45 +0200 Subject: [PATCH 2/3] remove 4.0.0-preview1 --- docs/modules/spark-k8s/partials/supported-versions.adoc | 1 - tests/test-definition.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 0a6fd493..4ec5fbdb 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,7 +3,6 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 4.0.0-preview1 (Hadoop 3.4.0, Scala 2.12, Python 3.11, Java 17) (experimental) - 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) - 3.4.3 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) - 3.4.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 2b3bc9c1..aada4df4 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -8,7 +8,6 @@ dimensions: - 3.4.2 - 3.4.3 - 3.5.1 - - 4.0.0-preview1 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev From 70b5e68c43455d2735f1f644ea6966efa4e75854 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:28:39 +0200 Subject: [PATCH 3/3] fix: iceberg version when using custom images --- .../kuttl/iceberg/10-deploy-spark-app.yaml.j2 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 index 89143cee..0311e042 100644 --- a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 @@ -41,8 +41,14 @@ spec: mountPath: /stackable/spark/jobs deps: packages: - # need to extract only the major and minor versions - - org.apache.iceberg:iceberg-spark-runtime-{{ test_scenario['values']['spark'].rsplit('.', maxsplit=1)[0] }}_2.12:1.4.0 + # + # The iceberg runtime contains the spark and scala versions in the form : + # + # ._. + # + # We extract the spark parts from the test scenario value. + # + - org.apache.iceberg:iceberg-spark-runtime-{{ ".".join(test_scenario['values']['spark'].split('.')[:2]) }}_2.12:1.5.2 volumes: - name: script configMap: