From ece42131765de2c79a841672ecd9a848e34bc55e Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sun, 29 Sep 2019 02:57:55 +0900 Subject: [PATCH] [SPARK-21914][FOLLOWUP][TEST-HADOOP3.2][TEST-JAVA11] Clone SparkSession per each function example ### What changes were proposed in this pull request? In the PR, I propose to clone Spark session per-each expression example. Examples can modify SQL settings, and can influence on each other if they run in the same Spark session in parallel. ### Why are the changes needed? This should fix test failures like [this](https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-maven-hadoop-3.2-jdk-11/478/testReport/junit/org.apache.spark.sql/SQLQuerySuite/check_outputs_of_expression_examples/) checking of the `Like` example: ``` org.apache.spark.sql.AnalysisException: the pattern '\%SystemDrive\%\Users%' is invalid, the escape character is not allowed to precede 'U'; at org.apache.spark.sql.catalyst.util.StringUtils$.fail$1(StringUtils.scala:48) at org.apache.spark.sql.catalyst.util.StringUtils$.escapeLikeRegex(StringUtils.scala:57) at org.apache.spark.sql.catalyst.expressions.Like.escape(regexpExpressions.scala:108) ``` ### Does this PR introduce any user-facing change? No ### How was this patch tested? By running `check outputs of expression examples` in `org.apache.spark.sql.SQLQuerySuite` Closes #25956 from MaxGekk/fix-expr-examples-checks. Authored-by: Maxim Gekk Signed-off-by: HyukjinKwon --- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 4a5201735fbda..46ef7943739ca 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -169,7 +169,9 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession { withSQLConf(SQLConf.UTC_TIMESTAMP_FUNC_ENABLED.key -> "true") { spark.sessionState.functionRegistry.listFunction().par.foreach { funcId => - val info = spark.sessionState.catalog.lookupFunctionInfo(funcId) + // Examples can change settings. We clone the session to prevent tests clashing. + val clonedSpark = spark.cloneSession() + val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId) val className = info.getClassName if (!ignoreSet.contains(className)) { withClue(s"Function '${info.getName}', Expression class '$className'") { @@ -177,7 +179,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession { checkExampleSyntax(example) example.split(" > ").toList.foreach(_ match { case exampleRe(sql, output) => - val df = spark.sql(sql) + val df = clonedSpark.sql(sql) val actual = unindentAndTrim( hiveResultString(df.queryExecution.executedPlan).mkString("\n")) val expected = unindentAndTrim(output)