Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-37133][SQL] Add a config to optionally enforce ANSI reserved keywords #34403

Closed
wants to merge 4 commits into from
Closed

[SPARK-37133][SQL] Add a config to optionally enforce ANSI reserved keywords #34403

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7539217
Add a config to optionally enforce ANSI reserved keywords
cloud-fan Oct 27, 2021
0c0b477
Update AstBuilder.scala
cloud-fan Oct 27, 2021
9e6188c
Update ExpressionParserSuite.scala
cloud-fan Oct 27, 2021
4269fe8
update doc
cloud-fan Oct 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1728,7 +1728,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
}

override def visitCurrentLike(ctx: CurrentLikeContext): Expression = withOrigin(ctx) {
if (conf.ansiEnabled) {
if (conf.ansiEnabled && conf.enforceReservedKeywords) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need conf.enforceReservedKeywords since it implies ANSI is enabled.

ctx.name.getType match {
case SqlBaseParser.CURRENT_DATE =>
CurrentDate()
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ abstract class AbstractSqlParser extends ParserInterface with SQLConfHelper with
parser.addErrorListener(ParseErrorListener)
parser.legacy_setops_precedence_enabled = conf.setOpsPrecedenceEnforced
parser.legacy_exponent_literal_as_decimal_enabled = conf.exponentLiteralAsDecimalEnabled
parser.SQL_standard_keyword_behavior = conf.ansiEnabled
parser.SQL_standard_keyword_behavior = conf.enforceReservedKeywords

try {
try {
Expand Down
10 changes: 10 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2590,6 +2590,14 @@ object SQLConf {
.booleanConf
.createWithDefault(false)

val ENFORCE_RESERVED_KEYWORDS = buildConf("spark.sql.ansi.enforceReservedKeywords")
.doc(s"When true and '${ANSI_ENABLED.key}' is true, the Spark SQL parser enforces the ANSI " +
"reserved keywords and forbids SQL queries that use reserved keywords as alias names " +
"and/or identifiers for table, view, function, etc.")
.version("3.3.0")
.booleanConf
.createWithDefault(true)

val SORT_BEFORE_REPARTITION =
buildConf("spark.sql.execution.sortBeforeRepartition")
.internal()
Expand Down Expand Up @@ -4041,6 +4049,8 @@ class SQLConf extends Serializable with Logging {

def ansiEnabled: Boolean = getConf(ANSI_ENABLED)

def enforceReservedKeywords: Boolean = ansiEnabled && getConf(ENFORCE_RESERVED_KEYWORDS)

def timestampType: AtomicType = getConf(TIMESTAMP_TYPE) match {
case "TIMESTAMP_LTZ" =>
// For historical reason, the TimestampType maps to TIMESTAMP WITH LOCAL TIME ZONE
Expand Down
10 changes: 9 additions & 1 deletion sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -933,10 +933,18 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("current_timestamp", CurrentTimestamp())
}

withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
def testNonAnsiBehavior(): Unit = {
assertEqual("current_date", UnresolvedAttribute.quoted("current_date"))
assertEqual("current_timestamp", UnresolvedAttribute.quoted("current_timestamp"))
}
withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
gengliangwang marked this conversation as resolved.
Show resolved Hide resolved
testNonAnsiBehavior()
}
withSQLConf(
SQLConf.ANSI_ENABLED.key -> "true",
SQLConf.ENFORCE_RESERVED_KEYWORDS.key -> "false") {
testNonAnsiBehavior()
}
}

test("SPARK-36736: (NOT) ILIKE (ANY | SOME | ALL) expressions") {
Expand Down
9 changes: 9 additions & 0 deletions ...lyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,15 @@ class TableIdentifierParserSuite extends SQLKeywordUtils {
assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.$keyword"))
}
}

withSQLConf(
SQLConf.ANSI_ENABLED.key -> "true",
SQLConf.ENFORCE_RESERVED_KEYWORDS.key -> "false") {
reservedKeywordsInAnsiMode.foreach { keyword =>
assert(TableIdentifier(keyword) === parseTableIdentifier(s"$keyword"))
assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.$keyword"))
}
}
}

test("table identifier - strict keywords") {
Expand Down