Make code compatible with spark 3.2 (#93)

tubular · Dec 17, 2022 · 7267a91 · 7267a91
1 parent 77e4c05
commit 7267a91
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 8 deletions.
diff --git a/sparkly/writer.py b/sparkly/writer.py
@@ -50,7 +50,7 @@ class SparklyWriter(object):
     """
     def __init__(self, df):
         self._df = df
-        self._spark = df.sparkSession
+        self._spark = df.sql_ctx.sparkSession
 
     def by_url(self, url):
         """Write a dataframe to a destination specified by `url`.

diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
@@ -215,15 +215,15 @@ def test_type_mismatch(self):
                 {'f1': T.IntegerType()},
             )
 
-        with self.assertRaisesRegex(TypeError, r'f1 is IntegerType\(\), expected LongType\(\)'):
+        with self.assertRaisesRegex(TypeError, r'f1 is IntegerType\(?\)?, expected LongType\(?\)?'):
             schema_has(
                 T.StructType([T.StructField('f1', T.IntegerType())]),
                 T.StructType([T.StructField('f1', T.LongType())]),
             )
 
         with self.assertRaisesRegex(
             TypeError,
-            r'f1\.element\.s1 is IntegerType\(\), expected LongType\(\)',
+            r'f1\.element\.s1 is IntegerType\(?\)?, expected LongType\(?\)?',
         ):
             schema_has(
                 T.StructType([
@@ -242,20 +242,26 @@ def test_type_mismatch(self):
 
         with self.assertRaisesRegex(
             TypeError,
-            r'element is IntegerType\(\), expected LongType\(\)',
+            r'element is IntegerType\(?\)?, expected LongType\(?\)?',
         ):
             schema_has(
                 T.ArrayType(T.IntegerType()),
                 T.ArrayType(T.LongType()),
             )
 
-        with self.assertRaisesRegex(TypeError, r'key is StringType\(\), expected LongType\(\)'):
+        with self.assertRaisesRegex(
+            TypeError,
+            r'key is StringType\(?\)?, expected LongType\(?\)?',
+        ):
             schema_has(
                 T.MapType(T.StringType(), T.IntegerType()),
                 T.MapType(T.LongType(), T.IntegerType()),
             )
 
-        with self.assertRaisesRegex(TypeError, r'value is IntegerType\(\), expected LongType\(\)'):
+        with self.assertRaisesRegex(
+            TypeError,
+            r'value is IntegerType\(?\)?, expected LongType\(?\)?'
+        ):
             schema_has(
                 T.MapType(T.StringType(), T.IntegerType()),
                 T.MapType(T.StringType(), T.LongType()),
@@ -286,7 +292,7 @@ def test_undefined_field(self):
 
         with self.assertRaisesRegex(
             TypeError,
-            r'element is IntegerType\(\), expected LongType\(\)',
+            r'element is IntegerType\(?\)?, expected LongType\(?\)?',
         ):
             schema_has(
                 T.ArrayType(T.IntegerType()),

diff --git a/tox.ini b/tox.ini
@@ -15,7 +15,15 @@
 #
 
 [tox]
-envlist = spark33,no_extras,docs
+envlist = spark32,spark33,no_extras,docs
+
+[testenv:spark32]
+commands = py.test --cov=sparkly --cov-report term-missing tests/integration tests/unit
+deps =
+    -rrequirements.txt
+    -rrequirements_dev.txt
+    -rrequirements_extras.txt
+    pyspark==3.2.3
 
 [testenv:spark33]
 commands = py.test --cov=sparkly --cov-report term-missing tests/integration tests/unit