From 1e24cefbd2e3d9e42fc3a5efa96394270a7078bd Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 19 Apr 2022 13:13:24 +0300 Subject: [PATCH 1/9] Double quote SQL statements --- .../scala/org/apache/spark/sql/errors/QueryErrorsBase.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala index 4708b47cfb890..7daf8ae7325db 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.errors +import java.util.Locale + import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.util.quoteIdentifier import org.apache.spark.sql.types.{DataType, DoubleType, FloatType} @@ -48,7 +50,7 @@ trait QueryErrorsBase { // Quote sql statements in error messages. def toSQLStmt(text: String): String = { - s"'$text'" + "\"" + text.toUpperCase(Locale.ROOT) + "\"" } def toSQLId(parts: Seq[String]): String = { From 4aea02139b19190bae288a1e6896d8e63efddc6b Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 19 Apr 2022 13:25:53 +0300 Subject: [PATCH 2/9] Apply toSQLStmt to QueryExecutionErrors --- .../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 4 ++-- .../apache/spark/sql/errors/QueryExecutionErrorsSuite.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 79e36cb485a97..b0d00962a9270 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -1884,13 +1884,13 @@ object QueryExecutionErrors extends QueryErrorsBase { def repeatedPivotsUnsupportedError(): Throwable = { new SparkUnsupportedOperationException( errorClass = "UNSUPPORTED_FEATURE", - messageParameters = Array("Repeated pivots.")) + messageParameters = Array(s"Repeated ${toSQLStmt("pivot")}s.")) } def pivotNotAfterGroupByUnsupportedError(): Throwable = { new SparkUnsupportedOperationException( errorClass = "UNSUPPORTED_FEATURE", - messageParameters = Array("Pivot not after a groupBy.")) + messageParameters = Array(s"${toSQLStmt("pivot")} not after a ${toSQLStmt("group by")}.")) } private val aesFuncName = toSQLId("aes_encrypt") + "/" + toSQLId("aes_decrypt") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala index 85956bd88768b..ea4673bd4d151 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala @@ -163,7 +163,7 @@ class QueryExecutionErrorsSuite extends QueryTest } assert(e1.getErrorClass === "UNSUPPORTED_FEATURE") assert(e1.getSqlState === "0A000") - assert(e1.getMessage === "The feature is not supported: Repeated pivots.") + assert(e1.getMessage === """The feature is not supported: Repeated "PIVOT"s.""") val e2 = intercept[SparkUnsupportedOperationException] { trainingSales @@ -174,7 +174,7 @@ class QueryExecutionErrorsSuite extends QueryTest } assert(e2.getErrorClass === "UNSUPPORTED_FEATURE") assert(e2.getSqlState === "0A000") - assert(e2.getMessage === "The feature is not supported: Pivot not after a groupBy.") + assert(e2.getMessage === """The feature is not supported: "PIVOT" not after a "GROUP BY".""") } test("INCONSISTENT_BEHAVIOR_CROSS_VERSION: " + From a4a83085ec2302356353c0cedb4fe7d0df711553 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 19 Apr 2022 14:52:35 +0300 Subject: [PATCH 3/9] Apply toSQLStmt to QueryParsingErrors --- .../sql/catalyst/parser/AstBuilder.scala | 2 +- .../spark/sql/errors/QueryParsingErrors.scala | 71 ++++++++++++++----- .../sql/errors/QueryParsingErrorsSuite.scala | 57 +++++++-------- 3 files changed, 78 insertions(+), 52 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index c435e319637bf..9b1a0d1f2dc92 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1166,7 +1166,7 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit } if (join.LATERAL != null) { if (!Seq(Inner, Cross, LeftOuter).contains(joinType)) { - throw QueryParsingErrors.unsupportedLateralJoinTypeError(ctx, joinType.toString) + throw QueryParsingErrors.unsupportedLateralJoinTypeError(ctx, joinType.sql) } LateralJoin(left, LateralSubquery(plan(join.right)), joinType, condition) } else { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala index b3785b2062365..9eab4421bbe5c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala @@ -90,13 +90,19 @@ object QueryParsingErrors extends QueryErrorsBase { } def transformNotSupportQuantifierError(ctx: ParserRuleContext): Throwable = { - new ParseException("UNSUPPORTED_FEATURE", - Array("TRANSFORM does not support DISTINCT/ALL in inputs"), ctx) + new ParseException( + errorClass = "UNSUPPORTED_FEATURE", + messageParameters = Array(s"${toSQLStmt("TRANSFORM")} does not support" + + s" ${toSQLStmt("DISTINCT")}/${toSQLStmt("ALL")} in inputs"), + ctx) } def transformWithSerdeUnsupportedError(ctx: ParserRuleContext): Throwable = { - new ParseException("UNSUPPORTED_FEATURE", - Array("TRANSFORM with serde is only supported in hive mode"), ctx) + new ParseException( + errorClass = "UNSUPPORTED_FEATURE", + messageParameters = Array( + s"${toSQLStmt("TRANSFORM")} with serde is only supported in hive mode"), + ctx) } def lateralWithPivotInFromClauseNotAllowedError(ctx: FromClauseContext): Throwable = { @@ -104,19 +110,31 @@ object QueryParsingErrors extends QueryErrorsBase { } def lateralJoinWithNaturalJoinUnsupportedError(ctx: ParserRuleContext): Throwable = { - new ParseException("UNSUPPORTED_FEATURE", Array("LATERAL join with NATURAL join."), ctx) + new ParseException( + errorClass = "UNSUPPORTED_FEATURE", + messageParameters = Array(s"${toSQLStmt("LATERAL")} join with ${toSQLStmt("NATURAL")} join."), + ctx) } def lateralJoinWithUsingJoinUnsupportedError(ctx: ParserRuleContext): Throwable = { - new ParseException("UNSUPPORTED_FEATURE", Array("LATERAL join with USING join."), ctx) + new ParseException( + errorClass = "UNSUPPORTED_FEATURE", + messageParameters = Array(s"${toSQLStmt("LATERAL")} join with ${toSQLStmt("USING")} join."), + ctx) } def unsupportedLateralJoinTypeError(ctx: ParserRuleContext, joinType: String): Throwable = { - new ParseException("UNSUPPORTED_FEATURE", Array(s"LATERAL join type '$joinType'."), ctx) + new ParseException( + errorClass = "UNSUPPORTED_FEATURE", + messageParameters = Array(s"${toSQLStmt("LATERAL")} join type ${toSQLStmt(joinType)}."), + ctx) } def invalidLateralJoinRelationError(ctx: RelationPrimaryContext): Throwable = { - new ParseException("INVALID_SQL_SYNTAX", Array("LATERAL can only be used with subquery."), ctx) + new ParseException( + errorClass = "INVALID_SQL_SYNTAX", + messageParameters = Array(s"${toSQLStmt("LATERAL")} can only be used with subquery."), + ctx) } def repetitiveWindowDefinitionError(name: String, ctx: WindowClauseContext): Throwable = { @@ -135,7 +153,7 @@ object QueryParsingErrors extends QueryErrorsBase { } def naturalCrossJoinUnsupportedError(ctx: RelationContext): Throwable = { - new ParseException("UNSUPPORTED_FEATURE", Array("NATURAL CROSS JOIN."), ctx) + new ParseException("UNSUPPORTED_FEATURE", Array(toSQLStmt("NATURAL CROSS JOIN") + "."), ctx) } def emptyInputForTableSampleError(ctx: ParserRuleContext): Throwable = { @@ -301,15 +319,18 @@ object QueryParsingErrors extends QueryErrorsBase { } def showFunctionsUnsupportedError(identifier: String, ctx: IdentifierContext): Throwable = { - new ParseException("INVALID_SQL_SYNTAX", - Array(s"SHOW ${toSQLId(identifier)} FUNCTIONS not supported"), ctx) + new ParseException( + errorClass = "INVALID_SQL_SYNTAX", + messageParameters = Array( + s"${toSQLStmt("SHOW")} ${toSQLId(identifier)} ${toSQLStmt("FUNCTIONS")} not supported"), + ctx) } def showFunctionsInvalidPatternError(pattern: String, ctx: ParserRuleContext): Throwable = { new ParseException( errorClass = "INVALID_SQL_SYNTAX", messageParameters = Array( - s"Invalid pattern in SHOW FUNCTIONS: ${toSQLId(pattern)}. " + + s"Invalid pattern in ${toSQLStmt("SHOW FUNCTIONS")}: ${toSQLId(pattern)}. " + s"It must be a ${toSQLType(StringType)} literal."), ctx) } @@ -416,13 +437,20 @@ object QueryParsingErrors extends QueryErrorsBase { } def createFuncWithBothIfNotExistsAndReplaceError(ctx: CreateFunctionContext): Throwable = { - new ParseException("INVALID_SQL_SYNTAX", - Array("CREATE FUNCTION with both IF NOT EXISTS and REPLACE is not allowed."), ctx) + new ParseException( + errorClass = "INVALID_SQL_SYNTAX", + messageParameters = Array( + s"${toSQLStmt("CREATE FUNCTION")} with both ${toSQLStmt("IF NOT EXISTS")} " + + s"and ${toSQLStmt("REPLACE")} is not allowed."), + ctx) } def defineTempFuncWithIfNotExistsError(ctx: CreateFunctionContext): Throwable = { - new ParseException("INVALID_SQL_SYNTAX", - Array("It is not allowed to define a TEMPORARY function with IF NOT EXISTS."), ctx) + new ParseException( + errorClass = "INVALID_SQL_SYNTAX", + messageParameters = Array( + s"It is not allowed to define a ${toSQLStmt("TEMPORARY")} function" + + s" with ${toSQLStmt("IF NOT EXISTS")}."), ctx) } def unsupportedFunctionNameError(funcName: Seq[String], ctx: CreateFunctionContext): Throwable = { @@ -435,7 +463,8 @@ object QueryParsingErrors extends QueryErrorsBase { ctx: CreateFunctionContext): Throwable = { new ParseException( "INVALID_SQL_SYNTAX", - Array("Specifying a database in CREATE TEMPORARY FUNCTION is not allowed: " + + Array( + s"Specifying a database in ${toSQLStmt("CREATE TEMPORARY FUNCTION")} is not allowed: " + toSQLId(databaseName)), ctx) } @@ -449,8 +478,12 @@ object QueryParsingErrors extends QueryErrorsBase { } def invalidNameForDropTempFunc(name: Seq[String], ctx: ParserRuleContext): Throwable = { - new ParseException("INVALID_SQL_SYNTAX", - Array(s"DROP TEMPORARY FUNCTION requires a single part name but got: ${toSQLId(name)}"), ctx) + new ParseException( + errorClass = "INVALID_SQL_SYNTAX", + messageParameters = Array( + s"${toSQLStmt("DROP TEMPORARY FUNCTION")} requires a single part name but got: " + + toSQLId(name)), + ctx) } def defaultColumnNotImplementedYetError(ctx: ParserRuleContext): Throwable = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala index 225d4f33b4179..37b6f0f1641f4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala @@ -21,6 +21,8 @@ import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.test.SharedSparkSession +// Turn of the length check because most of the tests check entire error messages +// scalastyle:off line.size.limit class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { def validateParsingError( sqlText: String, @@ -42,7 +44,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "0A000", message = """ - |The feature is not supported: LATERAL join with NATURAL join.(line 1, pos 14) + |The feature is not supported: "LATERAL" join with "NATURAL" join.(line 1, pos 14) | |== SQL == |SELECT * FROM t1 NATURAL JOIN LATERAL (SELECT c1 + c2 AS c2) @@ -57,7 +59,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "0A000", message = """ - |The feature is not supported: LATERAL join with USING join.(line 1, pos 14) + |The feature is not supported: "LATERAL" join with "USING" join.(line 1, pos 14) | |== SQL == |SELECT * FROM t1 JOIN LATERAL (SELECT c1 + c2 AS c2) USING (c2) @@ -66,21 +68,17 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { } test("UNSUPPORTED_FEATURE: Unsupported LATERAL join type") { - Seq( - ("RIGHT OUTER", "RightOuter"), - ("FULL OUTER", "FullOuter"), - ("LEFT SEMI", "LeftSemi"), - ("LEFT ANTI", "LeftAnti")).foreach { pair => + Seq("RIGHT OUTER", "FULL OUTER", "LEFT SEMI", "LEFT ANTI").foreach { joinType => validateParsingError( - sqlText = s"SELECT * FROM t1 ${pair._1} JOIN LATERAL (SELECT c1 + c2 AS c3) ON c2 = c3", + sqlText = s"SELECT * FROM t1 $joinType JOIN LATERAL (SELECT c1 + c2 AS c3) ON c2 = c3", errorClass = "UNSUPPORTED_FEATURE", sqlState = "0A000", message = s""" - |The feature is not supported: LATERAL join type '${pair._2}'.(line 1, pos 14) + |The feature is not supported: "LATERAL" join type "$joinType".(line 1, pos 14) | |== SQL == - |SELECT * FROM t1 ${pair._1} JOIN LATERAL (SELECT c1 + c2 AS c3) ON c2 = c3 + |SELECT * FROM t1 $joinType JOIN LATERAL (SELECT c1 + c2 AS c3) ON c2 = c3 |--------------^^^ |""".stripMargin) } @@ -101,7 +99,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "42000", message = s""" - |Invalid SQL syntax: LATERAL can only be used with subquery.(line 1, pos $pos) + |Invalid SQL syntax: "LATERAL" can only be used with subquery.(line 1, pos $pos) | |== SQL == |$sqlText @@ -117,7 +115,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "0A000", message = """ - |The feature is not supported: NATURAL CROSS JOIN.(line 1, pos 14) + |The feature is not supported: "NATURAL CROSS JOIN".(line 1, pos 14) | |== SQL == |SELECT * FROM a NATURAL CROSS JOIN b @@ -177,8 +175,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "0A000", message = """ - |The feature is not supported: """.stripMargin + - """TRANSFORM does not support DISTINCT/ALL in inputs(line 1, pos 17) + |The feature is not supported: "TRANSFORM" does not support "DISTINCT"/"ALL" in inputs(line 1, pos 17) | |== SQL == |SELECT TRANSFORM(DISTINCT a) USING 'a' FROM t @@ -194,12 +191,10 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "0A000", message = """ - |The feature is not supported: """.stripMargin + - """TRANSFORM with serde is only supported in hive mode(line 1, pos 0) + |The feature is not supported: "TRANSFORM" with serde is only supported in hive mode(line 1, pos 0) | |== SQL == - |SELECT TRANSFORM(a) ROW FORMAT SERDE """.stripMargin + - """'org.apache.hadoop.hive.serde2.OpenCSVSerde' USING 'a' FROM t + |SELECT TRANSFORM(a) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' USING 'a' FROM t |^^^ |""".stripMargin) } @@ -241,7 +236,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "42000", message = """ - |Invalid SQL syntax: SHOW `sys` FUNCTIONS not supported(line 1, pos 5) + |Invalid SQL syntax: "SHOW" `sys` "FUNCTIONS" not supported(line 1, pos 5) | |== SQL == |SHOW sys FUNCTIONS @@ -249,7 +244,6 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { |""".stripMargin) } - // scalastyle:off line.size.limit test("INVALID_SQL_SYNTAX: Invalid pattern in show functions") { validateParsingError( sqlText = "SHOW FUNCTIONS IN db f1", @@ -257,7 +251,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "42000", message = s""" - |Invalid SQL syntax: Invalid pattern in SHOW FUNCTIONS: `f1`. It must be a STRING literal.(line 1, pos 21) + |Invalid SQL syntax: Invalid pattern in "SHOW FUNCTIONS": `f1`. It must be a STRING literal.(line 1, pos 21) | |== SQL == |SHOW FUNCTIONS IN db f1 @@ -269,14 +263,13 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { sqlState = "42000", message = s""" - |Invalid SQL syntax: Invalid pattern in SHOW FUNCTIONS: `f1`. It must be a STRING literal.(line 1, pos 26) - | - |== SQL == - |SHOW FUNCTIONS IN db LIKE f1 - |--------------------------^^^ - |""".stripMargin) + |Invalid SQL syntax: Invalid pattern in "SHOW FUNCTIONS": `f1`. It must be a STRING literal.(line 1, pos 26) + | + |== SQL == + |SHOW FUNCTIONS IN db LIKE f1 + |--------------------------^^^ + |""".stripMargin) } - // scalastyle:on line.size.limit test("INVALID_SQL_SYNTAX: Create function with both if not exists and replace") { val sqlText = @@ -286,7 +279,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { |JAR '/path/to/jar2' |""".stripMargin val errorDesc = - "CREATE FUNCTION with both IF NOT EXISTS and REPLACE is not allowed.(line 2, pos 0)" + """"CREATE FUNCTION" with both "IF NOT EXISTS" and "REPLACE" is not allowed.(line 2, pos 0)""" validateParsingError( sqlText = sqlText, @@ -313,7 +306,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { |JAR '/path/to/jar2' |""".stripMargin val errorDesc = - "It is not allowed to define a TEMPORARY function with IF NOT EXISTS.(line 2, pos 0)" + """It is not allowed to define a "TEMPORARY" function with "IF NOT EXISTS".(line 2, pos 0)""" validateParsingError( sqlText = sqlText, @@ -365,7 +358,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { |JAR '/path/to/jar2' |""".stripMargin val errorDesc = - "Specifying a database in CREATE TEMPORARY FUNCTION is not allowed: `db`(line 2, pos 0)" + """Specifying a database in "CREATE TEMPORARY FUNCTION" is not allowed: `db`(line 2, pos 0)""" validateParsingError( sqlText = sqlText, @@ -386,7 +379,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { test("INVALID_SQL_SYNTAX: Drop temporary function requires a single part name") { val errorDesc = - "DROP TEMPORARY FUNCTION requires a single part name but got: `db`.`func`(line 1, pos 0)" + "\"DROP TEMPORARY FUNCTION\" requires a single part name but got: `db`.`func`(line 1, pos 0)" validateParsingError( sqlText = "DROP TEMPORARY FUNCTION db.func", From fdc7e69424cd2679fb902a3b38a64ce74a8b52d1 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 19 Apr 2022 15:07:31 +0300 Subject: [PATCH 4/9] Apply toSQLStmt to QueryCompilationErrors --- .../apache/spark/sql/errors/QueryCompilationErrors.scala | 3 ++- .../spark/sql/errors/QueryCompilationErrorsSuite.scala | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 26c8cebe0d532..f939a0929647c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -1573,7 +1573,8 @@ object QueryCompilationErrors extends QueryErrorsBase { new AnalysisException( errorClass = "UNSUPPORTED_FEATURE", messageParameters = Array( - s"Using PythonUDF in join condition of join type $joinType is not supported")) + "Using PythonUDF in join condition of join type " + + s"${toSQLStmt(joinType.sql)} is not supported.")) } def conflictingAttributesInJoinConditionError( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index f2088ed4449c3..e4d3d589b5e34 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -155,7 +155,7 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession { assert(e.getSqlState === "0A000") assert(e.message === "The feature is not supported: " + - "Using PythonUDF in join condition of join type LeftOuter is not supported") + "Using PythonUDF in join condition of join type \"LEFT OUTER\" is not supported.") } test("UNSUPPORTED_FEATURE: Using pandas UDF aggregate expression with pivot") { @@ -333,7 +333,8 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession { ) assert(e.getErrorClass === "FORBIDDEN_OPERATION") assert(e.message === - s"The operation 'DESC PARTITION' is not allowed on the temporary view: `$tempViewName`") + s"""The operation "DESC PARTITION" is not allowed """ + + s"on the temporary view: `$tempViewName`") } } } @@ -358,7 +359,8 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession { ) assert(e.getErrorClass === "FORBIDDEN_OPERATION") assert(e.message === - s"The operation 'DESC PARTITION' is not allowed on the view: `$viewName`") + s"""The operation "DESC PARTITION" is not allowed """ + + s"on the view: `$viewName`") } } } From 5883ed57499049dd748ac1b050522b75a87ddde2 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 19 Apr 2022 15:17:40 +0300 Subject: [PATCH 5/9] Apply toSQLStmt to QueryCompilationErrors --- .../org/apache/spark/sql/errors/QueryCompilationErrors.scala | 3 ++- .../spark/sql/errors/QueryCompilationErrorsDSv2Suite.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index f939a0929647c..dac6a6e731f39 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -95,7 +95,8 @@ object QueryCompilationErrors extends QueryErrorsBase { new AnalysisException( errorClass = "UNSUPPORTED_FEATURE", messageParameters = Array( - s"IF NOT EXISTS for the table ${toSQLId(tableName)} by INSERT INTO.")) + s"${toSQLStmt("IF NOT EXISTS")} for the table ${toSQLId(tableName)} " + + s"by ${toSQLStmt("INSERT INTO")}.")) } def nonPartitionColError(partitionName: String): Throwable = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsDSv2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsDSv2Suite.scala index d58c2b5629309..6f1178566f28d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsDSv2Suite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsDSv2Suite.scala @@ -52,7 +52,7 @@ class QueryCompilationErrorsDSv2Suite checkAnswer(spark.table(tbl), spark.emptyDataFrame) assert(e.getMessage === "The feature is not supported: " + - s"IF NOT EXISTS for the table `testcat`.`ns1`.`ns2`.`tbl` by INSERT INTO.") + s""""IF NOT EXISTS" for the table `testcat`.`ns1`.`ns2`.`tbl` by "INSERT INTO".""") assert(e.getErrorClass === "UNSUPPORTED_FEATURE") assert(e.getSqlState === "0A000") } From b7bee2fd0cd307b291e22c6b5d4222833709f0ae Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 19 Apr 2022 19:16:32 +0300 Subject: [PATCH 6/9] Adjust tests --- python/pyspark/sql/tests/test_udf.py | 12 ++++++------ .../ExtractPythonUDFFromJoinConditionSuite.scala | 4 ++-- .../spark/sql/catalyst/parser/PlanParserSuite.scala | 2 +- .../resources/sql-tests/results/describe.sql.out | 4 ++-- .../resources/sql-tests/results/join-lateral.sql.out | 4 ++-- .../resources/sql-tests/results/transform.sql.out | 4 ++-- .../execution/SparkScriptTransformationSuite.scala | 2 +- .../spark/sql/execution/command/DDLParserSuite.scala | 4 ++-- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py index 805d5a8dfec9a..1b1f9309e1b5c 100644 --- a/python/pyspark/sql/tests/test_udf.py +++ b/python/pyspark/sql/tests/test_udf.py @@ -258,15 +258,15 @@ def test_udf_not_supported_in_join_condition(self): def runWithJoinType(join_type, type_string): with self.assertRaisesRegex( AnalysisException, - "Using PythonUDF in join condition of join type %s is not supported" % type_string, + """Using PythonUDF in join condition of join type "%s" is not supported""" % type_string, ): left.join(right, [f("a", "b"), left.a1 == right.b1], join_type).collect() - runWithJoinType("full", "FullOuter") - runWithJoinType("left", "LeftOuter") - runWithJoinType("right", "RightOuter") - runWithJoinType("leftanti", "LeftAnti") - runWithJoinType("leftsemi", "LeftSemi") + runWithJoinType("full", "FULL OUTER") + runWithJoinType("left", "LEFT OUTER") + runWithJoinType("right", "RIGHT OUTER") + runWithJoinType("leftanti", "LEFT ANTI") + runWithJoinType("leftsemi", "LEFT SEMI") def test_udf_as_join_condition(self): left = self.spark.createDataFrame([Row(a=1, a1=1, a2=1), Row(a=2, a1=2, a2=2)]) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala index 0a85eb3e88015..fb1f413955d05 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala @@ -187,9 +187,9 @@ class ExtractPythonUDFFromJoinConditionSuite extends PlanTest { condition = Some(unevaluableJoinCond)) Optimize.execute(query.analyze) } - assert(e.message.contentEquals( + assert(e.message == "The feature is not supported: " + - s"Using PythonUDF in join condition of join type $joinType is not supported")) + s"""Using PythonUDF in join condition of join type "${joinType.sql}" is not supported.""") val query2 = testRelationLeft.join( testRelationRight, diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index 3d0d2fea7be0b..c825e259dcd0c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -1250,7 +1250,7 @@ class PlanParserSuite extends AnalysisTest { | "escapeChar" = "\\") |FROM testData """.stripMargin, - "TRANSFORM with serde is only supported in hive mode") + "\"TRANSFORM\" with serde is only supported in hive mode") } diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index c0db04fa5c86b..eefd308a9b2e0 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -462,7 +462,7 @@ DESC temp_v PARTITION (c='Us', d=1) struct<> -- !query output org.apache.spark.sql.AnalysisException -The operation 'DESC PARTITION' is not allowed on the temporary view: `temp_v` +The operation "DESC PARTITION" is not allowed on the temporary view: `temp_v` -- !query @@ -539,7 +539,7 @@ DESC v PARTITION (c='Us', d=1) struct<> -- !query output org.apache.spark.sql.AnalysisException -The operation 'DESC PARTITION' is not allowed on the view: `v` +The operation "DESC PARTITION" is not allowed on the view: `v` -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out b/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out index cc1619813dd55..6e47579a9b011 100644 --- a/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out @@ -153,7 +153,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -The feature is not supported: LATERAL join with NATURAL join.(line 1, pos 14) +The feature is not supported: "LATERAL" join with "NATURAL" join.(line 1, pos 14) == SQL == SELECT * FROM t1 NATURAL JOIN LATERAL (SELECT c1 + c2 AS c2) @@ -167,7 +167,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -The feature is not supported: LATERAL join with USING join.(line 1, pos 14) +The feature is not supported: "LATERAL" join with "USING" join.(line 1, pos 14) == SQL == SELECT * FROM t1 JOIN LATERAL (SELECT c1 + c2 AS c2) USING (c2) diff --git a/sql/core/src/test/resources/sql-tests/results/transform.sql.out b/sql/core/src/test/resources/sql-tests/results/transform.sql.out index be57390761ba3..69fe58e1343d1 100644 --- a/sql/core/src/test/resources/sql-tests/results/transform.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/transform.sql.out @@ -719,7 +719,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -The feature is not supported: TRANSFORM does not support DISTINCT/ALL in inputs(line 1, pos 17) +The feature is not supported: "TRANSFORM" does not support "DISTINCT"/"ALL" in inputs(line 1, pos 17) == SQL == SELECT TRANSFORM(DISTINCT b, a, c) @@ -739,7 +739,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -The feature is not supported: TRANSFORM does not support DISTINCT/ALL in inputs(line 1, pos 17) +The feature is not supported: "TRANSFORM" does not support "DISTINCT"/"ALL" in inputs(line 1, pos 17) == SQL == SELECT TRANSFORM(ALL b, a, c) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala index 5638743b7633d..1f431e173b3c7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala @@ -56,7 +56,7 @@ class SparkScriptTransformationSuite extends BaseScriptTransformationSuite with |FROM v """.stripMargin) }.getMessage - assert(e.contains("TRANSFORM with serde is only supported in hive mode")) + assert(e.contains("\"TRANSFORM\" with serde is only supported in hive mode")) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala index 05378e3229661..44694e10c6fcf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala @@ -469,9 +469,9 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession { DropFunctionCommand(Seq("a").asFunctionIdentifier, true, true)) intercept("DROP TEMPORARY FUNCTION a.b", - "DROP TEMPORARY FUNCTION requires a single part name") + "\"DROP TEMPORARY FUNCTION\" requires a single part name") intercept("DROP TEMPORARY FUNCTION IF EXISTS a.b", - "DROP TEMPORARY FUNCTION requires a single part name") + "\"DROP TEMPORARY FUNCTION\" requires a single part name") } test("SPARK-32374: create temporary view with properties not allowed") { From 27ac62767ed3a1e75b7bd2d2c726a7634936c81e Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Tue, 19 Apr 2022 21:49:34 +0300 Subject: [PATCH 7/9] Reformat test_udf.py --- python/pyspark/sql/tests/test_udf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py index 1b1f9309e1b5c..e40c3ba0d6479 100644 --- a/python/pyspark/sql/tests/test_udf.py +++ b/python/pyspark/sql/tests/test_udf.py @@ -258,7 +258,8 @@ def test_udf_not_supported_in_join_condition(self): def runWithJoinType(join_type, type_string): with self.assertRaisesRegex( AnalysisException, - """Using PythonUDF in join condition of join type "%s" is not supported""" % type_string, + """Using PythonUDF in join condition of join type "%s" is not supported""" + % type_string, ): left.join(right, [f("a", "b"), left.a1 == right.b1], join_type).collect() From 269b41d4ce0c525d36345ce8511505ff24cc03f2 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Wed, 20 Apr 2022 08:46:57 +0300 Subject: [PATCH 8/9] double quote temporary function --- .../scala/org/apache/spark/sql/errors/QueryParsingErrors.scala | 2 +- .../org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala index 9eab4421bbe5c..0245205cffb74 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala @@ -449,7 +449,7 @@ object QueryParsingErrors extends QueryErrorsBase { new ParseException( errorClass = "INVALID_SQL_SYNTAX", messageParameters = Array( - s"It is not allowed to define a ${toSQLStmt("TEMPORARY")} function" + + s"It is not allowed to define a ${toSQLStmt("TEMPORARY FUNCTION")}" + s" with ${toSQLStmt("IF NOT EXISTS")}."), ctx) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala index 37b6f0f1641f4..dc1ed2c9021f4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala @@ -306,7 +306,7 @@ class QueryParsingErrorsSuite extends QueryTest with SharedSparkSession { |JAR '/path/to/jar2' |""".stripMargin val errorDesc = - """It is not allowed to define a "TEMPORARY" function with "IF NOT EXISTS".(line 2, pos 0)""" + """It is not allowed to define a "TEMPORARY FUNCTION" with "IF NOT EXISTS".(line 2, pos 0)""" validateParsingError( sqlText = sqlText, From 7c6d78b7a4f8a57c24dfb1ba457a677745701048 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Wed, 20 Apr 2022 13:05:13 +0300 Subject: [PATCH 9/9] Trigger build