new error class

apache · Dec 7, 2023 · 9fefb05 · 9fefb05
1 parent cc29d6c
commit 9fefb05
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 28 deletions.
diff --git a/python/pyspark/errors/error_classes.py b/python/pyspark/errors/error_classes.py
@@ -84,14 +84,14 @@
       "Argument `<arg_name>` cannot be None."
     ]
   },
-  "CANNOT_CONFIGURE_SPARK_CONNECT": {
+  "CANNOT_CONFIGURE_SPARK_CONNECT_MASTER": {
     "message": [
-      "Spark Connect server cannot be configured with Spark master; however, found URL for Spark master [<url>]."
+      "Spark Connect server and Spark master cannot be configured together: Spark master [<master_url>], Spark Connect [<connect_url>]."
     ]
   },
-  "CANNOT_CONFIGURE_SPARK_MASTER": {
+  "CANNOT_CONFIGURE_SPARK_CONNECT": {
     "message": [
-      "Spark master cannot be configured with Spark Connect server; however, found URL for Spark Connect [<url>]."
+      "Spark Connect server cannot be configured: Existing [<existing_url>], New [<new_url>]."
     ]
   },
   "CANNOT_CONVERT_COLUMN_INTO_BOOL": {

diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
@@ -306,22 +306,16 @@ def _validate_startup_urls(
             Helper function that validates the combination of startup URLs and raises an exception
             if incompatible options are selected.
             """
-            if "spark.master" in self._options and (
+            if ("spark.master" in self._options or "MASTER" in os.environ) and (
                 "spark.remote" in self._options or "SPARK_REMOTE" in os.environ
             ):
                 raise PySparkRuntimeError(
-                    error_class="CANNOT_CONFIGURE_SPARK_MASTER",
+                    error_class="CANNOT_CONFIGURE_SPARK_CONNECT_MASTER",
                     message_parameters={
-                        "url": self._options.get("spark.remote", os.environ.get("SPARK_REMOTE"))
-                    },
-                )
-            if "spark.remote" in self._options and (
-                "spark.master" in self._options or "MASTER" in os.environ
-            ):
-                raise PySparkRuntimeError(
-                    error_class="CANNOT_CONFIGURE_SPARK_CONNECT",
-                    message_parameters={
-                        "url": self._options.get("spark.master", os.environ.get("MASTER"))
+                        "master_url": self._options.get("spark.master", os.environ.get("MASTER")),
+                        "connect_url": self._options.get(
+                            "spark.remote", os.environ.get("SPARK_REMOTE")
+                        ),
                     },
                 )
 
@@ -333,8 +327,8 @@ def _validate_startup_urls(
                     raise PySparkRuntimeError(
                         error_class="CANNOT_CONFIGURE_SPARK_CONNECT",
                         message_parameters={
-                            "new_url": os.environ["SPARK_REMOTE"],
-                            "existing_url": remote,
+                            "existing_url": os.environ["SPARK_REMOTE"],
+                            "new_url": remote,
                         },
                     )
 

diff --git a/python/pyspark/sql/tests/test_session.py b/python/pyspark/sql/tests/test_session.py
@@ -20,10 +20,11 @@
 import unittest.mock
 
 from pyspark import SparkConf, SparkContext
+from pyspark.errors import PySparkRuntimeError
 from pyspark.sql import SparkSession, SQLContext, Row
 from pyspark.sql.functions import col
 from pyspark.testing.sqlutils import ReusedSQLTestCase
-from pyspark.testing.utils import PySparkTestCase
+from pyspark.testing.utils import PySparkTestCase, PySparkErrorTestUtils
 
 
 class SparkSessionTests(ReusedSQLTestCase):
@@ -259,7 +260,7 @@ def test_get_sqlcontext_with_stopped_sparkcontext(self):
         self.assertIs(SQLContext.getOrCreate(self.sc)._sc, self.sc)
 
 
-class SparkSessionBuilderTests(unittest.TestCase):
+class SparkSessionBuilderTests(unittest.TestCase, PySparkErrorTestUtils):
     def test_create_spark_context_first_then_spark_session(self):
         sc = None
         session = None
@@ -353,21 +354,30 @@ def test_create_spark_context_with_initial_session_options_bool(self):
                 session.stop()
 
     def test_create_spark_context_with_invalid_configs(self):
-        with self.assertRaisesRegex(
-            RuntimeError,
-            "Spark master cannot be configured with Spark "
-            "Connect server; however, found URL for Spark Connect",
-        ):
+        with self.assertRaises(PySparkRuntimeError) as pe1:
             SparkSession.builder.config(map={"spark.master": "x", "spark.remote": "y"})
 
+        self.check_error(
+            exception=pe1.exception,
+            error_class="CANNOT_CONFIGURE_SPARK_CONNECT_MASTER",
+            message_parameters={"master_url": "x", "connect_url": "y"},
+        )
+
         with unittest.mock.patch.dict(
             "os.environ", {"SPARK_REMOTE": "remote_url", "SPARK_LOCAL_REMOTE": "true"}
         ):
-            with self.assertRaisesRegex(
-                RuntimeError, "Only one Spark Connect client URL can be set"
-            ):
+            with self.assertRaises(PySparkRuntimeError) as pe2:
                 SparkSession.builder.config("spark.remote", "different_remote_url")
 
+            self.check_error(
+                exception=pe2.exception,
+                error_class="CANNOT_CONFIGURE_SPARK_CONNECT",
+                message_parameters={
+                    "existing_url": "remote_url",
+                    "new_url": "different_remote_url",
+                },
+            )
+
 
 class SparkExtensionsTest(unittest.TestCase):
     # These tests are separate because it uses 'spark.sql.extensions' which is