Skip to content

Commit

Permalink
Added pushdown for findInSet string expression
Browse files Browse the repository at this point in the history
Summary: Added pushdown to singestore-spark-connetor for FindInSet string expression

Test Plan: https://webapp.io/memsql/commits?query=repo%3Asinglestore-spark-connector+id%3A232

Reviewers: amakarovych-ua

Reviewed By: amakarovych-ua

Subscribers: engineering-list

JIRA Issues: PLAT-6082

Differential Revision: https://grizzly.internal.memcompute.com/D55566
  • Loading branch information
PolliO committed Mar 22, 2022
1 parent 185098b commit 49b19c7
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 0 deletions.
10 changes: 10 additions & 0 deletions src/main/scala/com/singlestore/spark/ExpressionGen.scala
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,16 @@ object ExpressionGen extends LazyLogging {
if trimStr == UTF8String.fromString(" ") =>
f("RTRIM", srcStr)

case FindInSet(expressionExtractor(left), utf8StringFoldableExtractor(right)) => {
val str_array = right.toString.split(',')
var caseBranches = stringToJoinable("")
for (i <- 1 to str_array.length) {
caseBranches += Raw(s"WHEN '${str_array(i - 1)}'")
caseBranches += Raw(s"THEN '${i.toString}'")
}
block(Raw("CASE") + left + caseBranches + Raw("ELSE 0 END"))
}

// TODO: case _: Levenshtein => None

// ----------------------------------
Expand Down
29 changes: 29 additions & 0 deletions src/test/scala/com/singlestore/spark/SQLPushdownTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3047,6 +3047,35 @@ class SQLPushdownTest extends IntegrationSuiteBase with BeforeAndAfterEach with
}
}

describe("FindInSet") {
it("works") {
testQuery(
"select id, find_in_set(id, '82, 1,13,54,28,39,42, owns_house,120') from users")
}
it("constant example") {
testQuery(
"select id, find_in_set('39', '1,2,3990, 13,28,39,42,54,82,120') from users")
}
it("works with empty left argument") {
testQuery("select find_in_set('', '1,2,3') from users")
}
it("works with empty right argument") {
testQuery("select find_in_set(id, '') from users")
}
it("udf in the left argument") {
testQuery("select find_in_set(stringIdentity(critic_review), 'id') from movies",
expectPartialPushdown = true)
}
it("udf in the right argument") {
testQuery("select find_in_set(critic_review, stringIdentity('id')) from movies",
expectPartialPushdown = true)
}
it("joinable object in the right argument") {
testQuery("select find_in_set(critic_review, id) from movies",
expectPartialPushdown = true)
}
}

describe("StringTrim") {
it("works") {
testQuery("select id, trim(first_name) from users")
Expand Down

0 comments on commit 49b19c7

Please sign in to comment.