From 347dddd2bf028ca511daabe9dfba816e81ea0a42 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 8 Dec 2022 14:48:30 -0800 Subject: [PATCH] [SPARK-41454][PYTHON] Support Python 3.11 --- python/pyspark/shuffle.py | 9 ++++++--- python/setup.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/pyspark/shuffle.py b/python/pyspark/shuffle.py index 35c3397de503b..da03110c32197 100644 --- a/python/pyspark/shuffle.py +++ b/python/pyspark/shuffle.py @@ -78,9 +78,12 @@ def _get_local_dirs(sub): path = os.environ.get("SPARK_LOCAL_DIRS", "/tmp") dirs = path.split(",") if len(dirs) > 1: - # different order in different processes and instances - rnd = random.Random(os.getpid() + id(dirs)) - random.shuffle(dirs, rnd.random) + if sys.version_info < (3, 11): + # different order in different processes and instances + rnd = random.Random(os.getpid() + id(dirs)) + random.shuffle(dirs, rnd.random) + else: + random.shuffle(dirs) return [os.path.join(d, "python", str(os.getpid()), sub) for d in dirs] diff --git a/python/setup.py b/python/setup.py index af102f230839c..65db3912efebb 100755 --- a/python/setup.py +++ b/python/setup.py @@ -282,6 +282,7 @@ def run(self): 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Typing :: Typed'],