Update spark

jeroenvdhoven · Jun 12, 2024 · 2cf7976 · 2cf7976
1 parent 97d39a0
commit 2cf7976
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
@@ -25,8 +25,8 @@ jobs:
 
       - uses: vemonet/setup-spark@v1
         with:
-          spark-version: '3.2.1'
-          hadoop-version: '3.2'
+          spark-version: '3.4.0'
+          hadoop-version: '3'
 
       - name: Install dependencies
         run: make local-install

diff --git a/mlpype-spark/setup.py b/mlpype-spark/setup.py
@@ -6,8 +6,8 @@
     deps = [
         f"mlpype-base=={version}",
         # We will provide absolute no guarantees that our integration will work with
-        # EVERY version of pyspark. This has been developed under pyspark==3.2.1.
-        "pyspark>=3.2.1",
+        # EVERY version of pyspark. This has been developed under pyspark==3.4.0.
+        "pyspark>=3.4.0",
     ]
     strict_deps = [s.replace(">=", "==") for s in deps]
     test_deps = ["pandas"]