From e3731e8dcb570dc3f5274570bf0ba6c84a839976 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 15 Jan 2026 15:32:12 -0800 Subject: [PATCH 1/4] Fix test_arrow_c_stream when pandas/arrow does not exist --- .../sql/tests/arrow/test_arrow_c_stream.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py index 9534db71bae6..c7fdee1d5fe4 100644 --- a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py +++ b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py @@ -16,11 +16,25 @@ # import ctypes import unittest -import pyarrow as pa -import pandas as pd import pyspark.pandas as ps +from pyspark.testing.utils import ( + have_pandas, + have_pyarrow, + pandas_requirement_message, + pyarrow_requirement_message, +) +if have_pyarrow: + import pyarrow as pa +if have_pandas: + import pandas as pd + + +@unittest.skipIf( + not have_pyarrow or not have_pandas, + pyarrow_requirement_message or pandas_requirement_message, # type: ignore[arg-type] +) class TestSparkArrowCStreamer(unittest.TestCase): def test_spark_arrow_c_streamer_arrow_consumer(self): pdf = pd.DataFrame([[1, "a"], [2, "b"], [3, "c"], [4, "d"]], columns=["id", "value"]) From 5432cf2f870ea4244649f8e31faf0a73af4ee46f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 15 Jan 2026 15:53:10 -0800 Subject: [PATCH 2/4] Reorder --- python/pyspark/sql/tests/arrow/test_arrow_c_stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py index c7fdee1d5fe4..5726830e4d53 100644 --- a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py +++ b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py @@ -24,12 +24,12 @@ pyarrow_requirement_message, ) -if have_pyarrow: - import pyarrow as pa - if have_pandas: import pandas as pd +if have_pyarrow: + import pyarrow as pa + @unittest.skipIf( not have_pyarrow or not have_pandas, From d0321dfe45280b1e4e816b4cc3d31ee67e54d85c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 16 Jan 2026 11:11:29 -0800 Subject: [PATCH 3/4] Use pyspark.testing.main --- python/pyspark/sql/tests/arrow/test_arrow_c_stream.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py index 5726830e4d53..b4e07b03523a 100644 --- a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py +++ b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py @@ -67,12 +67,6 @@ def test_spark_arrow_c_streamer_arrow_consumer(self): if __name__ == "__main__": - from pyspark.sql.tests.arrow.test_arrow_c_stream import * # noqa: F401 + from pyspark.testing import main - try: - import xmlrunner # type: ignore - - test_runner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2) - except ImportError: - test_runner = None - unittest.main(testRunner=test_runner, verbosity=2) + main() From 25ef6f4ea06cbef596b50e77fce96a260586f08f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 16 Jan 2026 11:13:26 -0800 Subject: [PATCH 4/4] Remove unnecessary type ignore --- python/pyspark/sql/tests/arrow/test_arrow_c_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py index b4e07b03523a..e71d1d18329f 100644 --- a/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py +++ b/python/pyspark/sql/tests/arrow/test_arrow_c_stream.py @@ -33,7 +33,7 @@ @unittest.skipIf( not have_pyarrow or not have_pandas, - pyarrow_requirement_message or pandas_requirement_message, # type: ignore[arg-type] + pyarrow_requirement_message or pandas_requirement_message, ) class TestSparkArrowCStreamer(unittest.TestCase): def test_spark_arrow_c_streamer_arrow_consumer(self):