Skip to content

Commit

Permalink
Add tests to StringExpressionsSuite
Browse files Browse the repository at this point in the history
  • Loading branch information
@MaxGekk
MaxGekk committed Apr 3, 2020
1 parent 2b27f41 commit 0b53fe9
Showing 1 changed file with 44 additions and 0 deletions.
44 changes: 44 additions & 0 deletions ...yst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
import org.apache.spark.sql.types._

class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
Expand Down Expand Up @@ -155,6 +156,11 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(c1 endsWith "b", false, row)
checkEvaluation(c2 endsWith "b", null, row)
checkEvaluation(c1 endsWith Literal.create(null, StringType), null, row)

// Test escaping of arguments
GenerateUnsafeProjection.generate(Contains(Literal("\"quote"), Literal("\"quote")) :: Nil)
GenerateUnsafeProjection.generate(EndsWith(Literal("\"quote"), Literal("\"quote")) :: Nil)
GenerateUnsafeProjection.generate(StartsWith(Literal("\"quote"), Literal("\"quote")) :: Nil)
}

test("Substring") {
Expand Down Expand Up @@ -352,6 +358,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(Decode(b, Literal("utf-8")), null, create_row(null))
checkEvaluation(Decode(Literal.create(null, BinaryType), Literal("utf-8")), null)
checkEvaluation(Decode(b, Literal.create(null, StringType)), null, create_row(null))

// Test escaping of charset
GenerateUnsafeProjection.generate(Encode(a, Literal("\"quote")) :: Nil)
GenerateUnsafeProjection.generate(Decode(b, Literal("\"quote")) :: Nil)
}

test("initcap unit test") {
Expand Down Expand Up @@ -379,6 +389,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(Levenshtein(Literal("千世"), Literal("fog")), 3)
checkEvaluation(Levenshtein(Literal("世界千世"), Literal("大a界b")), 4)
// scalastyle:on

// Test escaping of arguments:
GenerateUnsafeProjection.generate(Levenshtein(Literal("\"quotea"), Literal("\"quoteb")) :: Nil)
}

test("soundex unit test") {
Expand Down Expand Up @@ -560,6 +573,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(StringTrim(Literal("a"), Literal.create(null, StringType)), null)
checkEvaluation(StringTrim(Literal.create(null, StringType), Literal("a")), null)

// Test escaping of arguments
GenerateUnsafeProjection.generate(StringTrim(Literal("\"quote"), Literal("\"quote")) :: Nil)

checkEvaluation(StringTrim(Literal("yxTomxx"), Literal("xyz")), "Tom")
checkEvaluation(StringTrim(Literal("xxxbarxxx"), Literal("x")), "bar")
}
Expand Down Expand Up @@ -587,6 +603,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(StringTrimLeft(Literal.create(null, StringType), Literal("a")), null)
checkEvaluation(StringTrimLeft(Literal("a"), Literal.create(null, StringType)), null)

// Test escaping of arguments
GenerateUnsafeProjection.generate(
StringTrimLeft(Literal("\"quote"), Literal("\"quote")) :: Nil)

checkEvaluation(StringTrimLeft(Literal("zzzytest"), Literal("xyz")), "test")
checkEvaluation(StringTrimLeft(Literal("zzzytestxyz"), Literal("xyz")), "testxyz")
checkEvaluation(StringTrimLeft(Literal("xyxXxyLAST WORD"), Literal("xy")), "XxyLAST WORD")
Expand Down Expand Up @@ -616,6 +636,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(StringTrimRight(Literal("a"), Literal.create(null, StringType)), null)
checkEvaluation(StringTrimRight(Literal.create(null, StringType), Literal("a")), null)

// Test escaping of arguments
GenerateUnsafeProjection.generate(
StringTrimRight(Literal("\"quote"), Literal("\"quote")) :: Nil)

checkEvaluation(StringTrimRight(Literal("testxxzx"), Literal("xyz")), "test")
checkEvaluation(StringTrimRight(Literal("xyztestxxzx"), Literal("xyz")), "xyztest")
checkEvaluation(StringTrimRight(Literal("TURNERyxXxy"), Literal("xy")), "TURNERyxX")
Expand All @@ -632,6 +656,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
FormatString(Literal("aa%d%s"), Literal.create(null, IntegerType), "cc"), "aanullcc")
checkEvaluation(
FormatString(Literal("aa%d%s"), 12, Literal.create(null, StringType)), "aa12null")

// Test escaping of arguments
GenerateUnsafeProjection.generate(FormatString(Literal("\"quote"), Literal("\"quote")) :: Nil)
}

test("SPARK-22603: FormatString should not generate codes beyond 64KB") {
Expand Down Expand Up @@ -662,6 +689,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(StringInstr(s1, s2), 1, create_row("花花世界", "花"))
checkEvaluation(StringInstr(s1, s2), 0, create_row("花花世界", "小"))
// scalastyle:on

// Test escaping of arguments
GenerateUnsafeProjection.generate(StringInstr(Literal("\"quote"), Literal("\"quote")) :: Nil)
}

test("LOCATE") {
Expand Down Expand Up @@ -718,6 +748,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(StringRPad(s1, s2, s3), null, row3)
checkEvaluation(StringRPad(s1, s2, s3), null, row4)
checkEvaluation(StringRPad(s1, s2, s3), null, row5)

// Test escaping of arguments
GenerateUnsafeProjection.generate(StringLPad(Literal("\"quote"), s2, Literal("\"quote")) :: Nil)
GenerateUnsafeProjection.generate(StringRPad(Literal("\"quote"), s2, Literal("\"quote")) :: Nil)
checkEvaluation(StringRPad(Literal("hi"), Literal(5)), "hi ")
checkEvaluation(StringRPad(Literal("hi"), Literal(1)), "h")
}
Expand All @@ -732,6 +766,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(StringRepeat(Literal("hi"), Literal(-1)), "", row1)
checkEvaluation(StringRepeat(s1, s2), "hihi", row1)
checkEvaluation(StringRepeat(s1, s2), null, row2)

// Test escaping of arguments
GenerateUnsafeProjection.generate(StringRepeat(Literal("\"quote"), Literal(2)) :: Nil)
}

test("REVERSE") {
Expand Down Expand Up @@ -897,6 +934,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
assert(ParseUrl(Seq(Literal("1"), Literal(2))).checkInputDataTypes().isFailure)
assert(ParseUrl(Seq(Literal(1), Literal("2"))).checkInputDataTypes().isFailure)
assert(ParseUrl(Seq(Literal("1"), Literal("2"), Literal(3))).checkInputDataTypes().isFailure)

// Test escaping of arguments
GenerateUnsafeProjection.generate(ParseUrl(Seq(Literal("\"quote"), Literal("\"quote"))) :: Nil)
}

test("Sentences") {
Expand All @@ -919,5 +959,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
answer)
checkEvaluation(Sentences("Hi there! The price was $1,234.56.... But, not now.", "XXX", "YYY"),
answer)

// Test escaping of arguments
GenerateUnsafeProjection.generate(
Sentences(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil)
}
}

0 comments on commit 0b53fe9

Please sign in to comment.