Merge pull request #1698 from moj-analytical-services/fix-spark-fixture

Fix spark fixture

tests/conftest.py

@@ -37,8 +37,7 @@ def pytest_collection_modifyitems(items, config):
                 item.add_marker(mark)
 
 
-@pytest.fixture(scope="module")
-def spark():
+def _make_spark():
     from pyspark import SparkConf, SparkContext
     from pyspark.sql import SparkSession
 
@@ -56,7 +55,12 @@ def spark():
 
     spark = SparkSession(sc)
     spark.sparkContext.setCheckpointDir("./tmp_checkpoints")
+    return spark
+
 
+@pytest.fixture(scope="module")
+def spark():
+    spark = _make_spark()
     yield spark
 
 
@@ -71,14 +75,14 @@ def df_spark(spark):
 # see e.g. https://stackoverflow.com/a/42400786/11811947
 # ruff: noqa: F811
 @pytest.fixture
-def test_helpers(spark, pg_engine):
+def test_helpers(pg_engine):
     # LazyDict to lazy-load helpers
     # That way we do not instantiate helpers we do not need
     # e.g. running only duckdb tests we don't need PostgresTestHelper
     # so we can run duckdb tests in environments w/o access to postgres
     return LazyDict(
         duckdb=(DuckDBTestHelper, []),
-        spark=(SparkTestHelper, [spark]),
+        spark=(SparkTestHelper, [_make_spark]),
         sqlite=(SQLiteTestHelper, []),
         postgres=(PostgresTestHelper, [pg_engine]),
     )

tests/helpers.py

@@ -105,8 +105,8 @@ def brl(self):
 
 
 class SparkTestHelper(TestHelper):
-    def __init__(self, spark):
-        self.spark = spark
+    def __init__(self, spark_creator_function):
+        self.spark = spark_creator_function()
 
     @property
     def Linker(self):