don't support binary types

helenyugithub · Apr 22, 2020 · f1f2afd · f1f2afd
1 parent d8696fe
commit f1f2afd
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -215,6 +215,7 @@ def test_vectorized_udf_string_in_udf(self):
     def test_pandas_python2_string(self):
         import pandas as pd
         self.spark.conf.set("spark.sql.execution.arrow.enabled", "true")
+        self.spark.conf.set("spark.sql.execution.arrow.fallback.enabled", "true")
         pdf = pd.DataFrame([['a', 'b']], columns = ["col1", "col2"])
         sdf = self.spark.createDataFrame(pdf)
         sdf2 = self.spark.createDataFrame([['a', 'b']], schema=['col1', 'col2'])
@@ -245,8 +246,8 @@ def test_vectorized_udf_datatype_string(self):
 
     def test_vectorized_udf_null_binary(self):
         import pyarrow as pa
-
-        if LooseVersion(pa.__version__) < LooseVersion("0.10.0"):
+        print(sys.version)
+        if LooseVersion(pa.__version__) < LooseVersion("0.10.0") or sys.version < '3':
             with QuietTest(self.sc):
                 with self.assertRaisesRegexp(
                         NotImplementedError,

diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
@@ -1609,6 +1609,8 @@ def to_arrow_type(dt):
         if LooseVersion(pa.__version__) < LooseVersion("0.10.0"):
             raise TypeError("Unsupported type in conversion to Arrow: " + str(dt) +
                             "\nPlease install pyarrow >= 0.10.0 for BinaryType support.")
+        if sys.version < '3':
+            raise TypeError("Unsupported type in conversion to Arrow for python2: " + str(dt))
         arrow_type = pa.binary()
     elif type(dt) == DateType:
         arrow_type = pa.date32()
@@ -1668,6 +1670,8 @@ def from_arrow_type(at):
         if LooseVersion(pa.__version__) < LooseVersion("0.10.0"):
             raise TypeError("Unsupported type in conversion from Arrow: " + str(at) +
                             "\nPlease install pyarrow >= 0.10.0 for BinaryType support.")
+        if sys.version < '3':
+            raise TypeError("Unsupported type in conversion to Arrow for python2: " + str(dt))
         spark_type = BinaryType()
     elif types.is_date32(at):
         spark_type = DateType()