-
Notifications
You must be signed in to change notification settings - Fork 28.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[SPARK-12576][SQL] Enable expression parsing in CatalystQl #10649
Changes from all commits
c15ae29
cd7f8ec
682df13
7f37d81
c2b35b7
b070bf9
bc0e298
3111ffb
beb5ca0
81588d2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,36 +17,157 @@ | |
|
||
package org.apache.spark.sql.catalyst | ||
|
||
import org.apache.spark.sql.AnalysisException | ||
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedAttribute, UnresolvedFunction} | ||
import org.apache.spark.sql.catalyst.expressions._ | ||
import org.apache.spark.sql.catalyst.plans.PlanTest | ||
import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation, Project} | ||
import org.apache.spark.unsafe.types.CalendarInterval | ||
|
||
class CatalystQlSuite extends PlanTest { | ||
val parser = new CatalystQl() | ||
|
||
test("test case insensitive") { | ||
val result = Project(UnresolvedAlias(Literal(1)):: Nil, OneRowRelation) | ||
assert(result === parser.parsePlan("seLect 1")) | ||
assert(result === parser.parsePlan("select 1")) | ||
assert(result === parser.parsePlan("SELECT 1")) | ||
} | ||
|
||
test("test NOT operator with comparison operations") { | ||
val parsed = parser.parsePlan("SELECT NOT TRUE > TRUE") | ||
val expected = Project( | ||
UnresolvedAlias( | ||
Not( | ||
GreaterThan(Literal(true), Literal(true))) | ||
) :: Nil, | ||
OneRowRelation) | ||
comparePlans(parsed, expected) | ||
} | ||
|
||
test("support hive interval literal") { | ||
def checkInterval(sql: String, result: CalendarInterval): Unit = { | ||
val parsed = parser.parsePlan(sql) | ||
val expected = Project( | ||
UnresolvedAlias( | ||
Literal(result) | ||
) :: Nil, | ||
OneRowRelation) | ||
comparePlans(parsed, expected) | ||
} | ||
|
||
def checkYearMonth(lit: String): Unit = { | ||
checkInterval( | ||
s"SELECT INTERVAL '$lit' YEAR TO MONTH", | ||
CalendarInterval.fromYearMonthString(lit)) | ||
} | ||
|
||
def checkDayTime(lit: String): Unit = { | ||
checkInterval( | ||
s"SELECT INTERVAL '$lit' DAY TO SECOND", | ||
CalendarInterval.fromDayTimeString(lit)) | ||
} | ||
|
||
def checkSingleUnit(lit: String, unit: String): Unit = { | ||
checkInterval( | ||
s"SELECT INTERVAL '$lit' $unit", | ||
CalendarInterval.fromSingleUnitString(unit, lit)) | ||
} | ||
|
||
checkYearMonth("123-10") | ||
checkYearMonth("496-0") | ||
checkYearMonth("-2-3") | ||
checkYearMonth("-123-0") | ||
|
||
checkDayTime("99 11:22:33.123456789") | ||
checkDayTime("-99 11:22:33.123456789") | ||
checkDayTime("10 9:8:7.123456789") | ||
checkDayTime("1 0:0:0") | ||
checkDayTime("-1 0:0:0") | ||
checkDayTime("1 0:0:1") | ||
|
||
for (unit <- Seq("year", "month", "day", "hour", "minute", "second")) { | ||
checkSingleUnit("7", unit) | ||
checkSingleUnit("-7", unit) | ||
checkSingleUnit("0", unit) | ||
} | ||
|
||
checkSingleUnit("13.123456789", "second") | ||
checkSingleUnit("-13.123456789", "second") | ||
} | ||
|
||
test("support scientific notation") { | ||
def assertRight(input: String, output: Double): Unit = { | ||
val parsed = parser.parsePlan("SELECT " + input) | ||
val expected = Project( | ||
UnresolvedAlias( | ||
Literal(output) | ||
) :: Nil, | ||
OneRowRelation) | ||
comparePlans(parsed, expected) | ||
} | ||
|
||
assertRight("9.0e1", 90) | ||
assertRight("0.9e+2", 90) | ||
assertRight("900e-1", 90) | ||
assertRight("900.0E-1", 90) | ||
assertRight("9.e+1", 90) | ||
|
||
intercept[AnalysisException](parser.parsePlan("SELECT .e3")) | ||
} | ||
|
||
test("parse expressions") { | ||
compareExpressions( | ||
parser.parseExpression("prinln('hello', 'world')"), | ||
UnresolvedFunction( | ||
"prinln", Literal("hello") :: Literal("world") :: Nil, false)) | ||
|
||
compareExpressions( | ||
parser.parseExpression("1 + r.r As q"), | ||
Alias(Add(Literal(1), UnresolvedAttribute("r.r")), "q")()) | ||
|
||
compareExpressions( | ||
parser.parseExpression("1 - f('o', o(bar))"), | ||
Subtract(Literal(1), | ||
UnresolvedFunction("f", | ||
Literal("o") :: | ||
UnresolvedFunction("o", UnresolvedAttribute("bar") :: Nil, false) :: | ||
Nil, false))) | ||
} | ||
|
||
test("table identifier") { | ||
assert(TableIdentifier("q") === parser.parseTableIdentifier("q")) | ||
assert(TableIdentifier("q", Some("d")) === parser.parseTableIdentifier("d.q")) | ||
intercept[AnalysisException](parser.parseTableIdentifier("")) | ||
// TODO parser swallows third identifier. | ||
// intercept[AnalysisException](parser.parseTableIdentifier("d.q.g")) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. are we going to support this? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think we should support this. Are there use cases for this? I'll create a fix, that'll throw an AnalysisException when we encounter this. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yea, throw exception seems reasonable to me |
||
} | ||
|
||
test("parse union/except/intersect") { | ||
parser.createPlan("select * from t1 union all select * from t2") | ||
parser.createPlan("select * from t1 union distinct select * from t2") | ||
parser.createPlan("select * from t1 union select * from t2") | ||
parser.createPlan("select * from t1 except select * from t2") | ||
parser.createPlan("select * from t1 intersect select * from t2") | ||
parser.createPlan("(select * from t1) union all (select * from t2)") | ||
parser.createPlan("(select * from t1) union distinct (select * from t2)") | ||
parser.createPlan("(select * from t1) union (select * from t2)") | ||
parser.createPlan("select * from ((select * from t1) union (select * from t2)) t") | ||
parser.parsePlan("select * from t1 union all select * from t2") | ||
parser.parsePlan("select * from t1 union distinct select * from t2") | ||
parser.parsePlan("select * from t1 union select * from t2") | ||
parser.parsePlan("select * from t1 except select * from t2") | ||
parser.parsePlan("select * from t1 intersect select * from t2") | ||
parser.parsePlan("(select * from t1) union all (select * from t2)") | ||
parser.parsePlan("(select * from t1) union distinct (select * from t2)") | ||
parser.parsePlan("(select * from t1) union (select * from t2)") | ||
parser.parsePlan("select * from ((select * from t1) union (select * from t2)) t") | ||
} | ||
|
||
test("window function: better support of parentheses") { | ||
parser.createPlan("select sum(product + 1) over (partition by ((1) + (product / 2)) " + | ||
parser.parsePlan("select sum(product + 1) over (partition by ((1) + (product / 2)) " + | ||
"order by 2) from windowData") | ||
parser.createPlan("select sum(product + 1) over (partition by (1 + (product / 2)) " + | ||
parser.parsePlan("select sum(product + 1) over (partition by (1 + (product / 2)) " + | ||
"order by 2) from windowData") | ||
parser.createPlan("select sum(product + 1) over (partition by ((product / 2) + 1) " + | ||
parser.parsePlan("select sum(product + 1) over (partition by ((product / 2) + 1) " + | ||
"order by 2) from windowData") | ||
|
||
parser.createPlan("select sum(product + 1) over (partition by ((product) + (1)) order by 2) " + | ||
parser.parsePlan("select sum(product + 1) over (partition by ((product) + (1)) order by 2) " + | ||
"from windowData") | ||
parser.createPlan("select sum(product + 1) over (partition by ((product) + 1) order by 2) " + | ||
parser.parsePlan("select sum(product + 1) over (partition by ((product) + 1) order by 2) " + | ||
"from windowData") | ||
parser.createPlan("select sum(product + 1) over (partition by (product + (1)) order by 2) " + | ||
parser.parsePlan("select sum(product + 1) over (partition by (product + (1)) order by 2) " + | ||
"from windowData") | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
can you add some function doc for these 3 functions, and also cover what the differences are?