Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-39905][SQL][TESTS] Remove checkErrorClass() and use checkError() instead #37322

Closed
wants to merge 7 commits into from
Closed

[SPARK-39905][SQL][TESTS] Remove checkErrorClass() and use checkError() instead #37322

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0335e55
Use instead of
MaxGekk Jul 27, 2022
c3874a7
Merge remote-tracking branch 'origin/master' into test-query-context
MaxGekk Jul 27, 2022
afac3d3
Fix QueryCompilationErrorsSuite
MaxGekk Jul 27, 2022
fbb3254
Merge remote-tracking branch 'origin/master' into test-query-context
MaxGekk Jul 28, 2022
dbcdc48
Map() -> Map.empty
MaxGekk Jul 28, 2022
b5e1c02
Add defaults to checkError()
MaxGekk Jul 28, 2022
513bb8b
Don't pass None
MaxGekk Jul 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
80 changes: 41 additions & 39 deletions sql/core/src/test/scala/org/apache/spark/sql/DatasetUnpivotSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,14 +305,17 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e,
errorClass = "UNPIVOT_VALUE_DATA_TYPE_MISMATCH",
msg = "Unpivot value columns must share a least common type, some types do not: \\[" +
"\"STRING\" \\(`str1#\\d+`\\), " +
"\"INT\" \\(`int1#\\d+`, `int2#\\d+`, `int3#\\d+`, ...\\), " +
"\"BIGINT\" \\(`long1#\\d+L`, `long2#\\d+L`\\)\\];(\n.*)*",
matchMsg = true)
errorSubClass = None,
sqlState = None,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does checkError provide default parameter value for errorSubClass and sqlState?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It has but if I remove settings of the parameters, I am getting the errors:

overloaded method value checkError with alternatives:
  (exception: org.apache.spark.SparkThrowable,errorClass: String,parameters: Map[String,String])Unit <and>
  (exception: org.apache.spark.SparkThrowable,errorClass: String,sqlState: String,parameters: Map[String,String])Unit <and>
  (exception: org.apache.spark.SparkThrowable,errorClass: String,errorSubClass: String,sqlState: String,parameters: Map[String,String])Unit <and>
  (exception: org.apache.spark.SparkThrowable,errorClass: String,errorSubClass: Option[String],sqlState: Option[String],parameters: Map[String,String],matchPVals: Boolean)Unit
 cannot be applied to (exception: org.apache.spark.sql.AnalysisException, errorClass: String, parameters: scala.collection.immutable.Map[String,String], matchPVals: Boolean)
    checkError(

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me add the overloaded method:

 protected def checkError(
      exception: SparkThrowable,
      errorClass: String,
      parameters: Map[String, String],
      matchPVals: Boolean): Unit =

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but I think it is better to set default values in the most wide method. I will remove one overloaded method and add default values.

parameters = Map(
"types" ->
(""""STRING" \(`str1#\d+`\), """ +
""""INT" \(`int1#\d+`, `int2#\d+`, `int3#\d+`, ...\), """ +
""""BIGINT" \(`long1#\d+L`, `long2#\d+L`\)""")),
matchPVals = true)
}

test("unpivot with compatible value types") {
Expand Down Expand Up @@ -358,12 +361,12 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e1,
errorClass = "UNRESOLVED_COLUMN",
msg = "A column or function parameter with name `1` cannot be resolved\\. " +
"Did you mean one of the following\\? \\[`id`, `int1`, `str1`, `str2`, `long1`\\];(\n.*)*",
matchMsg = true)
parameters = Map(
"objectName" -> "`1`",
"objectList" -> "`id`, `int1`, `str1`, `str2`, `long1`"))

// unpivoting where value column does not exist
val e2 = intercept[AnalysisException] {
Expand All @@ -374,12 +377,12 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e2,
errorClass = "UNRESOLVED_COLUMN",
msg = "A column or function parameter with name `does` cannot be resolved\\. " +
"Did you mean one of the following\\? \\[`id`, `int1`, `long1`, `str1`, `str2`\\];(\n.*)*",
matchMsg = true)
parameters = Map(
"objectName" -> "`does`",
"objectList" -> "`id`, `int1`, `long1`, `str1`, `str2`"))

// unpivoting with empty list of value columns
// where potential value columns are of incompatible types
Expand All @@ -391,14 +394,16 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e3,
errorClass = "UNPIVOT_VALUE_DATA_TYPE_MISMATCH",
msg = "Unpivot value columns must share a least common type, some types do not: \\[" +
"\"INT\" \\(`id#\\d+`, `int1#\\d+`\\), " +
"\"STRING\" \\(`str1#\\d+`, `str2#\\d+`\\), " +
"\"BIGINT\" \\(`long1#\\d+L`\\)\\];(\n.*)*",
matchMsg = true)
errorSubClass = None,
sqlState = None,
parameters = Map("types" ->
(""""INT" \(`id#\d+`, `int1#\d+`\), """ +
""""STRING" \(`str1#\d+`, `str2#\d+`\), """ +
""""BIGINT" \(`long1#\d+L`\)""")),
matchPVals = true)

// unpivoting with star id columns so that no value columns are left
val e4 = intercept[AnalysisException] {
Expand All @@ -409,12 +414,10 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e4,
errorClass = "UNPIVOT_REQUIRES_VALUE_COLUMNS",
msg = "At least one value column needs to be specified for UNPIVOT, " +
"all columns specified as ids;(\\n.*)*",
matchMsg = true)
parameters = Map())

// unpivoting with star value columns
// where potential value columns are of incompatible types
Expand All @@ -426,14 +429,16 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e5,
errorClass = "UNPIVOT_VALUE_DATA_TYPE_MISMATCH",
msg = "Unpivot value columns must share a least common type, some types do not: \\[" +
"\"INT\" \\(`id#\\d+`, `int1#\\d+`\\), " +
"\"STRING\" \\(`str1#\\d+`, `str2#\\d+`\\), " +
"\"BIGINT\" \\(`long1#\\d+L`\\)\\];(\n.*)*",
matchMsg = true)
errorSubClass = None,
sqlState = None,
parameters = Map("types" ->
(""""INT" \(`id#\d+`, `int1#\d+`\), """ +
""""STRING" \(`str1#\d+`, `str2#\d+`\), """ +
""""BIGINT" \(`long1#\d+L`\)""")),
matchPVals = true)

// unpivoting without giving values and no non-id columns
val e6 = intercept[AnalysisException] {
Expand All @@ -444,12 +449,10 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e6,
errorClass = "UNPIVOT_REQUIRES_VALUE_COLUMNS",
msg = "At least one value column needs to be specified for UNPIVOT, " +
"all columns specified as ids;(\\n.*)*",
matchMsg = true)
parameters = Map.empty)
}

test("unpivot after pivot") {
Expand Down Expand Up @@ -499,14 +502,13 @@ class DatasetUnpivotSuite extends QueryTest
valueColumnName = "val"
)
}
checkErrorClass(
checkError(
exception = e,
errorClass = "UNRESOLVED_COLUMN",
// expected message is wrong: https://issues.apache.org/jira/browse/SPARK-39783
msg = "A column or function parameter with name `an`\\.`id` cannot be resolved\\. " +
"Did you mean one of the following\\? " +
"\\[`an`.`id`, `int1`, `long1`, `str`.`one`, `str`.`two`\\];(\n.*)*",
matchMsg = true)
parameters = Map(
"objectName" -> "`an`.`id`",
"objectList" -> "`an`.`id`, `int1`, `long1`, `str`.`one`, `str`.`two`"))
}

test("unpivot with struct fields") {
Expand Down
62 changes: 21 additions & 41 deletions sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,15 @@ class QueryCompilationErrorsSuite

test("INVALID_PARAMETER_VALUE: the argument_index of string format is invalid") {
withSQLConf(SQLConf.ALLOW_ZERO_INDEX_IN_FORMAT_STRING.key -> "false") {
val e = intercept[AnalysisException] {
sql("select format_string('%0$s', 'Hello')")
}
checkErrorClass(
exception = e,
checkError(
exception = intercept[AnalysisException] {
sql("select format_string('%0$s', 'Hello')")
},
errorClass = "INVALID_PARAMETER_VALUE",
msg = "The value of parameter(s) 'strfmt' in `format_string` is invalid: " +
"expects %1$, %2$ and so on, but got %0$.; line 1 pos 7")
parameters = Map(
"parameter" -> "strfmt",
"functionName" -> "`format_string`",
"expected" -> "expects %1$, %2$ and so on, but got %0$."))
}
}

Expand Down Expand Up @@ -424,7 +425,7 @@ class QueryCompilationErrorsSuite
|order by a, b
|""".stripMargin), Row(1, 2) :: Nil)

checkErrorClass(
checkError(
exception = intercept[AnalysisException] {
sql(
"""select distinct struct.a, struct.b
Expand All @@ -436,41 +437,20 @@ class QueryCompilationErrorsSuite
|""".stripMargin)
},
errorClass = "UNRESOLVED_COLUMN",
msg = """A column or function parameter with name `struct`.`a` cannot be resolved. """ +
"""Did you mean one of the following\? \[`a`, `b`\]; line 6 pos 9;
|'Sort \['struct.a ASC NULLS FIRST, 'struct.b ASC NULLS FIRST\], true
|\+\- Distinct
| \+\- Project \[struct#\w+\.a AS a#\w+, struct#\w+\.b AS b#\w+\]
| \+\- SubqueryAlias tmp
| \+\- Union false, false
| :\- Project \[named_struct\(a, 1, b, 2, c, 3\) AS struct#\w+\]
| : \+\- OneRowRelation
| \+\- Project \[named_struct\(a, 1, b, 2, c, 4\) AS struct#\w+\]
| \+\- OneRowRelation
|""".stripMargin,
matchMsg = true)
parameters = Map("objectName" -> "`struct`.`a`", "objectList" -> "`a`, `b`"))
}

test("UNRESOLVED_COLUMN - SPARK-21335: support un-aliased subquery") {
withTempView("v") {
Seq(1 -> "a").toDF("i", "j").createOrReplaceTempView("v")
checkAnswer(sql("SELECT i from (SELECT i FROM v)"), Row(1))

checkErrorClass(
checkError(
exception = intercept[AnalysisException](sql("SELECT v.i from (SELECT i FROM v)")),
errorClass = "UNRESOLVED_COLUMN",
msg = "A column or function parameter with name `v`.`i` cannot be resolved. " +
"""Did you mean one of the following\? """ +
"""\[`__auto_generated_subquery_name`.`i`\]; line 1 pos 7;
|'Project \['v.i\]
|\+\- SubqueryAlias __auto_generated_subquery_name
| \+\- Project \[i#\w+\]
| \+\- SubqueryAlias v
| \+\- View \(`v`, \[i#\w+,j#\w+\]\)
| \+\- Project \[_\w+#\w+ AS i#\w+, _\w+#\w+ AS j#\w+\]
| \+\- LocalRelation \[_\w+#\w+, _\w+#\w+\]
|""".stripMargin,
matchMsg = true)
parameters = Map(
"objectName" -> "`v`.`i`",
"objectList" -> "`__auto_generated_subquery_name`.`i`"))

checkAnswer(sql("SELECT __auto_generated_subquery_name.i from (SELECT i FROM v)"), Row(1))
}
Expand Down Expand Up @@ -521,10 +501,10 @@ class QueryCompilationErrorsSuite
val e = intercept[AnalysisException] {
sql("ALTER TABLE t ADD COLUMNS (m.n int)")
}
checkErrorClass(
checkError(
exception = e,
errorClass = "INVALID_FIELD_NAME",
msg = "Field name `m`.`n` is invalid: `m` is not a struct.; line 1 pos 27")
parameters = Map("fieldName" -> "`m`.`n`", "path" -> "`m`"))
}
}

Expand Down Expand Up @@ -636,14 +616,14 @@ class QueryCompilationErrorsSuite
|LATERAL VIEW array_contains(value, 1) AS explodedvalue""".stripMargin).collect()
)

checkErrorClass(
checkError(
exception = e,
errorClass = "UNSUPPORTED_GENERATOR",
errorSubClass = Some("NOT_GENERATOR"),
msg = """The generator is not supported: `array_contains` is expected to be a generator. """ +
"However, its class is org.apache.spark.sql.catalyst.expressions.ArrayContains, " +
"which is not a generator.; line 4 pos 0"
)
sqlState = None,
parameters = Map(
"functionName" -> "`array_contains`",
"classCanonicalName" -> "org.apache.spark.sql.catalyst.expressions.ArrayContains"))
}
}

Expand Down
21 changes: 0 additions & 21 deletions sql/core/src/test/scala/org/apache/spark/sql/errors/QueryErrorsSuiteBase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,27 +22,6 @@ import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.test.SharedSparkSession

trait QueryErrorsSuiteBase extends SharedSparkSession {
def checkErrorClass(
exception: Exception with SparkThrowable,
errorClass: String,
errorSubClass: Option[String] = None,
msg: String,
sqlState: Option[String] = None,
matchMsg: Boolean = false): Unit = {
assert(exception.getErrorClass === errorClass)
sqlState.foreach(state => exception.getSqlState === state)
val fullErrorClass = if (errorSubClass.isDefined) {
errorClass + "." + errorSubClass.get
} else {
errorClass
}
if (matchMsg) {
assert(exception.getMessage.matches(s"""\\[$fullErrorClass\\] """ + msg),
"exception is: " + exception.getMessage)
} else {
assert(exception.getMessage === s"""[$fullErrorClass] """ + msg)
}
}

def validateParsingError(
sqlText: String,
Expand Down