From ac740e18b2302a2581fdec1d2914b9c315a2610f Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Wed, 8 Apr 2020 16:56:51 +0900 Subject: [PATCH] Show a better error message for different python and pip installation mistake --- python/pyspark/find_spark_home.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/pyspark/find_spark_home.py b/python/pyspark/find_spark_home.py index 9c4ed46598632..64b25f81a6224 100755 --- a/python/pyspark/find_spark_home.py +++ b/python/pyspark/find_spark_home.py @@ -40,6 +40,7 @@ def is_spark_home(path): paths = ["../", os.path.dirname(os.path.realpath(__file__))] # Add the path of the PySpark module if it exists + is_error_raised = False if sys.version < "3": import imp try: @@ -49,7 +50,7 @@ def is_spark_home(path): paths.append(os.path.join(module_home, "../../")) except ImportError: # Not pip installed no worries - pass + is_error_raised = True else: from importlib.util import find_spec try: @@ -59,7 +60,7 @@ def is_spark_home(path): paths.append(os.path.join(module_home, "../../")) except ImportError: # Not pip installed no worries - pass + is_error_raised = True # Normalize the paths paths = [os.path.abspath(p) for p in paths] @@ -68,6 +69,15 @@ def is_spark_home(path): return next(path for path in paths if is_spark_home(path)) except StopIteration: print("Could not find valid SPARK_HOME while searching {0}".format(paths), file=sys.stderr) + if is_error_raised: + print( + "Did you install PySpark via a package manager such as PIP or Conda? If so, your " + "Python executable does not have the PySpark installed. It is possible your " + "Python executable does not point out your PIP. Please check your default " + "'python' and if you set PYSPARK_PYTHON and/or PYSPARK_DRIVER_PYTHON environment " + "variables, and see if you can import PySpark. If you cannot import, you can " + "install by using Python executable directly, for example, 'python -m pip " + "install pyspark'", file=sys.stderr) sys.exit(-1) if __name__ == "__main__":