Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-48160][SQL] Add collation support for XPATH expressions #46508

Closed
wants to merge 4 commits into from
Closed

[SPARK-48160][SQL] Add collation support for XPATH expressions #46508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
467e2ea
Initial commit
uros-db May 9, 2024
95ba2e9
Merge branch 'apache:master' into xpath-expressions
uros-db May 10, 2024
d812f3c
Tests
uros-db May 13, 2024
412766b
Update xpath.scala
uros-db May 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 7 additions & 4 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.Cast._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.util.GenericArrayData
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.types.StringTypeAnyCollation
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String

Expand All @@ -39,15 +41,16 @@ abstract class XPathExtract
/** XPath expressions are always nullable, e.g. if the xml string is empty. */
override def nullable: Boolean = true

override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType)
override def inputTypes: Seq[AbstractDataType] =
Seq(StringTypeAnyCollation, StringTypeAnyCollation)

override def checkInputDataTypes(): TypeCheckResult = {
if (!path.foldable) {
DataTypeMismatch(
errorSubClass = "NON_FOLDABLE_INPUT",
messageParameters = Map(
"inputName" -> toSQLId("path"),
"inputType" -> toSQLType(StringType),
"inputType" -> toSQLType(StringTypeAnyCollation),
"inputExpr" -> toSQLExpr(path)
)
)
Expand Down Expand Up @@ -221,7 +224,7 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
// scalastyle:on line.size.limit
case class XPathString(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath_string"
override def dataType: DataType = StringType
override def dataType: DataType = SQLConf.get.defaultStringType

override def nullSafeEval(xml: Any, path: Any): Any = {
val ret = xpathUtil.evalString(xml.asInstanceOf[UTF8String].toString, pathString)
Expand All @@ -245,7 +248,7 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract {
// scalastyle:on line.size.limit
case class XPathList(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath"
override def dataType: DataType = ArrayType(StringType, containsNull = false)
override def dataType: DataType = ArrayType(SQLConf.get.defaultStringType, containsNull = false)

override def nullSafeEval(xml: Any, path: Any): Any = {
val nodeList = xpathUtil.evalNodeList(xml.asInstanceOf[UTF8String].toString, pathString)
Expand Down
44 changes: 44 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/CollationSQLExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,50 @@ class CollationSQLExpressionsSuite
})
}

test("Support XPath expressions with collation") {
case class XPathTestCase(
xml: String,
xpath: String,
functionName: String,
collationName: String,
result: Any,
resultType: DataType
)

val testCases = Seq(
XPathTestCase("<a><b>1</b></a>", "a/b",
"xpath_boolean", "UTF8_BINARY", true, BooleanType),
XPathTestCase("<A><B>1</B><B>2</B></A>", "sum(A/B)",
"xpath_short", "UTF8_BINARY", 3, ShortType),
XPathTestCase("<a><b>3</b><b>4</b></a>", "sum(a/b)",
"xpath_int", "UTF8_BINARY_LCASE", 7, IntegerType),
XPathTestCase("<A><B>5</B><B>6</B></A>", "sum(A/B)",
"xpath_long", "UTF8_BINARY_LCASE", 11, LongType),
XPathTestCase("<a><b>7</b><b>8</b></a>", "sum(a/b)",
"xpath_float", "UNICODE", 15.0, FloatType),
XPathTestCase("<A><B>9</B><B>0</B></A>", "sum(A/B)",
"xpath_double", "UNICODE", 9.0, DoubleType),
XPathTestCase("<a><b>b</b><c>cc</c></a>", "a/c",
"xpath_string", "UNICODE_CI", "cc", StringType("UNICODE_CI")),
XPathTestCase("<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>", "a/b/text()",
"xpath", "UNICODE_CI", Array("b1", "b2", "b3"), ArrayType(StringType("UNICODE_CI")))
)

// Supported collations
testCases.foreach(t => {
val query =
s"""
|select ${t.functionName}('${t.xml}', '${t.xpath}')
|""".stripMargin
// Result & data type
withSQLConf(SqlApiConf.DEFAULT_COLLATION -> t.collationName) {
val testQuery = sql(query)
checkAnswer(testQuery, Row(t.result))
assert(testQuery.schema.fields.head.dataType.sameType(t.resultType))
}
})
}

test("Support StringSpace expression with collation") {
case class StringSpaceTestCase(
input: Int,
Expand Down