From b30ef7a57b2d359285f0d57a60248d7d63d834ac Mon Sep 17 00:00:00 2001 From: Norbert Schultz Date: Tue, 19 Jan 2021 11:12:31 +0100 Subject: [PATCH 1/3] [SPARK-34115][CORE] Check SPARK_TESTING as lazy val to avoid slowdowns with many environment variables. --- core/src/main/scala/org/apache/spark/util/Utils.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 5e68dcd9df7fc..2827733368e83 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1939,7 +1939,15 @@ private[spark] object Utils extends Logging { * Indicates whether Spark is currently running unit tests. */ def isTesting: Boolean = { - sys.env.contains("SPARK_TESTING") || sys.props.contains(IS_TESTING.key) + hasSparkTestingEnv || sys.props.contains(IS_TESTING.key) + } + + /** + * Indicates if SPARK_TESTING environment variable is present. + */ + private lazy val hasSparkTestingEnv: Boolean = { + // as lazy val because sys.env can be slow with many environment variables. + sys.env.contains("SPARK_TESTING") } /** From eaa25f7a8f13db489dd0db829605271bf70eb22d Mon Sep 17 00:00:00 2001 From: Norbert Schultz Date: Tue, 19 Jan 2021 11:59:05 +0100 Subject: [PATCH 2/3] Update core/src/main/scala/org/apache/spark/util/Utils.scala Co-authored-by: Hyukjin Kwon --- core/src/main/scala/org/apache/spark/util/Utils.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 2827733368e83..709d34f1c9195 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1939,15 +1939,10 @@ private[spark] object Utils extends Logging { * Indicates whether Spark is currently running unit tests. */ def isTesting: Boolean = { - hasSparkTestingEnv || sys.props.contains(IS_TESTING.key) + // Scala's `sys.env` creates a ton of garbage by constructing Scala immutable maps, so + // we directly use the Java APIs instead. + System.getenv("SPARK_TESTING") != null || System.getProperty(IS_TESTING.key) != null } - - /** - * Indicates if SPARK_TESTING environment variable is present. - */ - private lazy val hasSparkTestingEnv: Boolean = { - // as lazy val because sys.env can be slow with many environment variables. - sys.env.contains("SPARK_TESTING") } /** From b52e42769220fc8f4c0af2ae035472e039f2601c Mon Sep 17 00:00:00 2001 From: Norbert Schultz Date: Tue, 19 Jan 2021 12:00:29 +0100 Subject: [PATCH 3/3] Build fix --- core/src/main/scala/org/apache/spark/util/Utils.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 709d34f1c9195..13fcfe4aed023 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1943,7 +1943,6 @@ private[spark] object Utils extends Logging { // we directly use the Java APIs instead. System.getenv("SPARK_TESTING") != null || System.getProperty(IS_TESTING.key) != null } - } /** * Terminates a process waiting for at most the specified duration.