From deaba5ea1ac52a6534cb7d271fcce8b66f2d868c Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Thu, 6 Dec 2018 17:04:49 +0900 Subject: [PATCH 01/18] WIP: SQL Reserved/Non-Reserved Key Words --- docs/_data/menu-sql.yaml | 2 ++ ...sql-reserved-and-non-reserved-key-words.md | 24 +++++++++++++++++++ .../spark/sql/catalyst/parser/SqlBase.g4 | 9 +++++-- .../sql/catalyst/parser/AstBuilder.scala | 15 +++++++++++- .../parser/TableIdentifierParserSuite.scala | 11 ++++++++- 5 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 docs/sql-reserved-and-non-reserved-key-words.md diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml index cd065ea01dda4..5d321ee0ec11f 100644 --- a/docs/_data/menu-sql.yaml +++ b/docs/_data/menu-sql.yaml @@ -70,6 +70,8 @@ url: sql-migration-guide-upgrade.html - text: Compatibility with Apache Hive url: sql-migration-guide-hive-compatibility.html + - text: SQL Reserved/Non-Reserved Key Words + url: sql-reserved-and-non-reserved-key-words.html - text: Reference url: sql-reference.html subitems: diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md new file mode 100644 index 0000000000000..ec7f02535659a --- /dev/null +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -0,0 +1,24 @@ +--- +layout: global +title: SQL Reserved/Non-Reserved Key Words +displayTitle: SQL Reserved/Non-Reserved Key Words +--- + +Reserved key words in Spark SQL basically follow the ANSI SQL-2011 standards, but it is slightly different between each other. +A full list of reserved/non-reserved key words is as follows: + + + + + + + + + + + + + + + +
Key WordSpark SQLSQL-2011
ALLreservedreserved
ANYreservedreserved
XXXnon-reservedreserved
YYYreservednon-reserved
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index b39681d886c5c..8b7b10788df8b 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -719,7 +719,8 @@ qualifiedName ; identifier - : strictIdentifier + : reserved + | strictIdentifier | ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON | UNION | INTERSECT | EXCEPT | SETMINUS ; @@ -744,6 +745,10 @@ number | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral ; +reserved + : ALL | ANY + ; + nonReserved : SHOW | TABLES | COLUMNS | COLUMN | PARTITIONS | FUNCTIONS | DATABASES | ADD @@ -770,7 +775,7 @@ nonReserved | REVOKE | GRANT | LOCK | UNLOCK | MSCK | REPAIR | RECOVER | EXPORT | IMPORT | LOAD | VALUES | COMMENT | ROLE | ROLES | COMPACTIONS | PRINCIPALS | TRANSACTIONS | INDEX | INDEXES | LOCKS | OPTION | LOCAL | INPATH | ASC | DESC | LIMIT | RENAME | SETS - | AT | NULLS | OVERWRITE | ALL | ANY | ALTER | AS | BETWEEN | BY | CREATE | DELETE + | AT | NULLS | OVERWRITE | ANY | ALTER | AS | BETWEEN | BY | CREATE | DELETE | DESCRIBE | DROP | EXISTS | FALSE | FOR | GROUP | IN | INSERT | INTO | IS |LIKE | NULL | ORDER | OUTER | TABLE | TRUE | WITH | RLIKE | AND | CASE | CAST | DISTINCT | DIV | ELSE | END | FUNCTION | INTERVAL | MACRO | OR | STRATIFY | THEN diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 24bbe116ad897..e1cc7412a08c7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -942,7 +942,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ override def visitTableIdentifier( ctx: TableIdentifierContext): TableIdentifier = withOrigin(ctx) { - TableIdentifier(ctx.table.getText, Option(ctx.db).map(_.getText)) + TableIdentifier(visitIdentifier(ctx.table), Option(ctx.db).map(_.getText)) } /** @@ -1404,6 +1404,19 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } } + override def visitIdentifier(ctx: IdentifierContext): String = withOrigin(ctx) { + val keyword = ctx.getText + if (ctx.reserved() != null) { + // This option is expected to come from `spark.sql.parser.ansi.enabled` (pr20433): + // val ansi = conf.ansiParserEnabled + val ansi = true + if (ansi) { + throw new ParseException(s"'$keyword' is reserved in the ANSI SQL-2011 standard.", ctx) + } + } + keyword + } + /** * Create a [[CreateStruct]] expression. */ diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index ff0de0fb7c1f0..4ea2b1b9ebcc9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -42,7 +42,7 @@ class TableIdentifierParserSuite extends SparkFunSuite { "tblproperties", "temporary", "terminated", "tinyint", "touch", "transactions", "unarchive", "undo", "uniontype", "unlock", "unset", "unsigned", "uri", "use", "utc", "utctimestamp", "view", "while", "year", "work", "transaction", "write", "isolation", "level", "snapshot", - "autocommit", "all", "any", "alter", "array", "as", "authorization", "between", "bigint", + "autocommit", "alter", "array", "as", "authorization", "between", "bigint", "binary", "boolean", "both", "by", "create", "cube", "current_date", "current_timestamp", "cursor", "date", "decimal", "delete", "describe", "double", "drop", "exists", "external", "false", "fetch", "float", "for", "grant", "group", "grouping", "import", "in", @@ -76,6 +76,15 @@ class TableIdentifierParserSuite extends SparkFunSuite { assert(TableIdentifier("x.y.z", None) === parseTableIdentifier("`x.y.z`")) } + test("table identifier - reserved by the SQL-2011 standard") { + Seq("all", "any").foreach { keyword => + val errMsg = intercept[ParseException] { parseTableIdentifier(keyword) }.getMessage + assert(errMsg.contains(s"'$keyword' is reserved in the ANSI SQL-2011 standard.")) + assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) + assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) + } + } + test("table identifier - strict keywords") { // SQL Keywords. hiveStrictNonReservedKeyword.foreach { keyword => From c3b2f136022376017dacf4979a41f93bfae30126 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 30 Jan 2019 22:54:55 +0900 Subject: [PATCH 02/18] Fix --- ...sql-reserved-and-non-reserved-key-words.md | 568 +++++++++++++++++- .../spark/sql/catalyst/parser/SqlBase.g4 | 84 +-- .../sql/catalyst/parser/AstBuilder.scala | 8 +- .../sql/catalyst/parser/ParseDriver.scala | 4 +- .../apache/spark/sql/internal/SQLConf.scala | 8 + .../parser/TableIdentifierParserSuite.scala | 48 +- 6 files changed, 657 insertions(+), 63 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index ec7f02535659a..23b2cdcad3021 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -4,21 +4,565 @@ title: SQL Reserved/Non-Reserved Key Words displayTitle: SQL Reserved/Non-Reserved Key Words --- +When `spark.sql.parser.ansi.enabled` is set to true (false by default), some keywords are reserved for Spark SQL. Reserved key words in Spark SQL basically follow the ANSI SQL-2011 standards, but it is slightly different between each other. A full list of reserved/non-reserved key words is as follows: - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + non-reserved + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Key WordSpark SQLSQL-2011
ALLreservedreserved
ANYreservedreserved
XXXnon-reservedreserved
YYYreservednon-reserved
ABSreserved
ABSOLUTEnon-reserved
ACOSnon-reserved
ACTIONnon-reserved
ADDnon-reservednon-reserved
AFTERnon-reservednon-reserved
ALLreservedreserved
ALLOCATEreserved
ALTERnon-reservedreserved
ANALYZEnon-reservednon-reserved
ANDreservedreserved
ANYreservedreserved
AREreserved
ARCHIVEnon-reservednon-reserved
ARRAYnon-reservedreserved
ARRAY_AGGreserved
ARRAY_MAX_CARDINALITYreserved
ASreservedreserved
ASCnon-reservednon-reserved
ASENSITIVEreserved
ASINreserved
ASSERTIONnon-reserved
ASYMMETRICreserved
ATnon-reservedreserved
ATANnon-reserved
ATOMICreserved
AUTHORIZATIONreservedreserved
AVGreserved
BEFOREnon-reserved
BEGINreserved
BEGIN_FRAMEreserved
BEGIN_PARTITIONreserved
BETWEENnon-reservedreserved
BIGINTreserved
BINARYreserved
BITnon-reserved
BIT_LENGTHnon-reserved
BLOBreserved
BOOLEANreserved
BOTHreservedreserved
BREADTHnon-reserved
BUCKETnon-reservednon-reserved
BUCKETSnon-reservednon-reserved
BYnon-reservedreserved
CACHEnon-reservednon-reserved
CALLreserved
CALLEDreserved
CARDINALITYreserved
CASCADEnon-reservedreserved
CASCADEDreserved
CASEreservedreserved
CASTreservedreserved
CATALOGnon-reserved
CEILreserved
CEILINGreserved
CHANGEnon-reservednon-reserved
CHARreserved
CHAR_LENGTHreserved
CHARACTERreserved
CHARACTER_LENGTHreserved
CHECKreservedreserved
CLASSIFIERnon-reserved
CLEARnon-reservednon-reserved
CLOBreserved
CLOSEreserved
CLUSTERnon-reservednon-reserved
CLUSTEREDnon-reservednon-reserved
COALESCEreserved
CODEGENnon-reservednon-reserved
COLLATEreservedreserved
COLLATIONnon-reserved
COLLECTreserved
COLLECTIONnon-reservednon-reserved
COLUMNreservedreserved
COLUMNSnon-reservednon-reserved
COMMENTnon-reservednon-reserved
COMMITnon-reservedreserved
COMPACTnon-reservednon-reserved
COMPACTIONSnon-reservednon-reserved
COMPUTEnon-reservednon-reserved
CONCATENATEnon-reservednon-reserved
CONDITIONreserved
CONNECTnon-reserved
CONNECTIONnon-reserved
CONSTRAINTreservedreserved
CONSTRAINTSnon-reserved
CONSTRUCTORnon-reserved
CONTAINSnon-reserved
CONTINUEnon-reserved
CONVERTreserved
COPYnon-reserved
CORRreserved
CORRESPONDINGreserved
COSnon-reserved
COSHnon-reserved
COSTnon-reservednon-reserved
COUNTreserved
COVAR_POPreserved
COVAR_SAMPreserved
CREATEreservedreserved
CROSSreservedreserved
CUBEnon-reservedreserved
CUME_DISTreserved
CURRENTnon-reservedreserved
CURRENT_CATALOGreserved
CURRENT_DATEreservedreserved
CURRENT_DEFAULT_TRANSFORM_GROUPreserved
CURRENT_PATHreserved
CURRENT_ROLEreserved
CURRENT_ROWreserved
CURRENT_SCHEMAreserved
CURRENT_TIMEreservedreserved
CURRENT_TIMESTAMPreservedreserved
CURRENT_TRANSFORM_GROUP_FOR_TYPEreserved
CURRENT_USERreservedreserved
CURSORreserved
CYCLEreserved
DATAnon-reservednon-reserved
DATABASEnon-reservednon-reserved
DATABASESnon-reservednon-reserved
DATEreserved
DAYreserved
DBPROPERTIESnon-reservednon-reserved
DEALLOCATEreserved
DECreserved
DECFLOATnon-reserved
DECIMALreserved
DECLAREreserved
DEFAULTreserved
DEFERRABLEnon-reserved
DEFERREDnon-reserved
DEFINEnon-reserved
DEFINEDnon-reservednon-reserved
DELETEnon-reservedreserved
DELIMITEDnon-reservednon-reserved
DENSE_RANKreserved
DEPTHnon-reserved
DEREFreserved
DESCnon-reservednon-reserved
DESCRIBEnon-reservedreserved
DESCRIPTORnon-reserved
DETERMINISTICreserved
DFSnon-reservednon-reserved
DIAGNOSTICSnon-reserved
DIRECTORIESnon-reservednon-reserved
DIRECTORYnon-reservednon-reserved
DISCONNECTreserved
DISTINCTreservedreserved
DISTRIBUTEnon-reservednon-reserved
DIVnon-reservednon-reserved
DOreserved
DOMAINnon-reserved
DOUBLEreserved
DROPnon-reservedreserved
DYNAMICreserved
EACHreserved
ELEMENTreserved
ELSEreservedreserved
ELSEIFreserved
EMPTYnon-reserved
ENDreservedreserved
END_FRAMEreserved
END_PARTITIONreserved
ENDreserved
EXECreserved
EQUALSnon-reserved
ESCAPEreserved
ESCAPEDnon-reservednon-reserved
EVERYreserved
EXCEPTreservedreserved
EXCEPTIONnon-reserved
EXCHANGEnon-reservednon-reserved
EXECreserved
EXECUTEreserved
EXISTSnon-reservedreserved
EXITnon-reserved
EXPnon-reserved
EXPLAINnon-reservednon-reserved
EXPORTnon-reservednon-reserved
EXTENDEDnon-reservednon-reserved
EXTERNALnon-reservedreserved
EXTRACTnon-reservedreserved
FALSEreservedreserved
FETCHreservedreserved
FIELDSnon-reservednon-reserved
FILEFORMATnon-reservednon-reserved
FILTERreserved
FIRSTnon-reservednon-reserved
FIRST_VALUEreserved
FLOATreserved
FOLLOWINGnon-reservednon-reserved
FORreservedreserved
FOREIGNreservedreserved
FORMATnon-reservednon-reserved
FORMATTEDnon-reservednon-reserved
FOUNDnon-reserved
FRAME_ROWreserved
FREEreserved
FROMreservedreserved
FULLreservedreserved
FUNCTIONnon-reservedreserved
FUNCTIONSnon-reservednon-reserved
FUSIONnon-reserved
GENERALnon-reserved
GETreserved
GLOBALnon-reservedreserved
GOnon-reserved
GOTOnon-reserved
GRANTreservedreserved
GROUPreservedreserved
GROUPINGnon-reservedreserved
GROUPSreserved
HANDLERreserved
HAVINGreservedreserved
HOLDreserved
HOURreserved
IDENTITYreserved
IFnon-reservedreserved
IGNOREnon-reservednon-reserved
IMMEDIATEnon-reserved
IMPORTnon-reservednon-reserved
INreservedreserved
INDICATORreserved
INDEXnon-reservednon-reserved
INDEXESnon-reservednon-reserved
INITIALnon-reserved
INITIALLYnon-reserved
INNERreservedreserved
INOUTreserved
INPATHnon-reservednon-reserved
INPUTnon-reserved
INPUTFORMATnon-reservednon-reserved
INSENSITIVEreserved
INSERTnon-reservedreserved
INTreserved
INTEGERreserved
INTERSECTreservedreserved
INTERSECTIONreserved
INTERVALnon-reservedreserved
INTOreservedreserved
ISreservedreserved
ISOLATIONnon-reserved
ITEMSnon-reservednon-reserved
ITERATEreserved
JOINreservedreserved
JSON_ARRAYnon-reserved
JSON_ARRAYAGGnon-reserved
JSON_EXISTSnon-reserved
JSON_OBJECTnon-reserved
JSON_OBJECTAGGnon-reserved
JSON_QUERYnon-reserved
JSON_TABLEnon-reserved
JSON_TABLE_PRIMITIVEnon-reserved
JSON_VALUEnon-reserved
KEYnon-reserved
KEYSnon-reservednon-reserved
LAGnon-reserved
LANGUAGEreserved
LARGEreserved
LASTnon-reservednon-reserved
LAST_VALUEreserved
LATERALnon-reservedreserved
LAZYnon-reservednon-reserved
LEADreserved
LEADINGreservedreserved
LEAVEreserved
LEFTreservedreserved
LEVELnon-reserved
LIKEnon-reservedreserved
LIKE_REGEXreserved
LIMITnon-reservednon-reserved
LINESnon-reservednon-reserved
LISTnon-reservednon-reserved
LISTAGGnon-reserved
LNreserved
LOADnon-reservednon-reserved
LOCALnon-reservedreserved
LOCALTIMEreserved
LOCALTIMESTAMPreserved
LOCATIONnon-reservednon-reserved
LOCATORnon-reserved
LOCKnon-reservednon-reserved
LOCKSnon-reservednon-reserved
LOGnon-reserved
LOG10non-reserved
LOGICALnon-reservednon-reserved
LOOPreserved
LOWERreserved
MACROnon-reservednon-reserved
MAPnon-reservednon-reserved
MATCHreserved
MATCH_NUMBERnon-reserved
MATCH_RECOGNIZEnon-reserved
MATCHESnon-reserved
MAXreserved
MEMBERreserved
MERGEreserved
METHODreserved
MINreserved
MINUTEreserved
MODreserved
MODIFIESreserved
MODULEreserved
MONTHreserved
MSCKnon-reservednon-reserved
MULTISETreserved
NAMESnon-reserved
NATIONALreserved
NATURALreservedreserved
NCHARreserved
NCLOBreserved
NEWreserved
NEXTnon-reserved
NOnon-reservedreserved
NONEreserved
NORMALIZEreserved
NOTreservedreserved
NTH_VALUEreserved
NTILEreserved
NULLreservedreserved
NULLSnon-reservednon-reserved
NULLIFreserved
NUMERICreserved
OBJECTnon-reserved
OCCURRENCES_REGEXnon-reserved
OCTET_LENGTHreserved
OFnon-reservedreserved
OFFSETreserved
OLDreserved
OMITnon-reserved
ONreservedreserved
ONEnon-reserved
ONLYreservedreserved
OPENreserved
OPTIONnon-reservednon-reserved
OPTIONSnon-reservednon-reserved
ORreservedreserved
ORDERreservedreserved
ORDINALITYnon-reserved
OUTnon-reservedreserved
OUTERreservedreserved
OUTPUTnon-reserved
OUTPUTFORMATnon-reservednon-reserved
OVERnon-reservednon-reserved
OVERLAPSreservedreserved
OVERLAYreserved
OVERWRITEnon-reservednon-reserved
PADnon-reserved
PARAMETERreserved
PARTIALnon-reserved
PARTITIONnon-reservedreserved
PARTITIONEDnon-reservednon-reserved
PARTITIONSnon-reservednon-reserved
PATHnon-reserved
PATTERNnon-reserved
PERnon-reserved
PERCENTreserved
PERCENT_RANKreserved
PERCENTILE_CONTreserved
PERCENTILE_DISCreserved
PERCENTLITnon-reservednon-reserved
PERIODreserved
PIVOTnon-reservednon-reserved
PORTIONreserved
POSITIONnon-reservedreserved
POSITION_REGEXreserved
POWERreserved
PRECEDESreserved
PRECEDINGnon-reservednon-reserved
PRECISIONreserved
PREPAREreserved
PRESERVEnon-reserved
PRIMARYreservedreserved
PRINCIPALSnon-reservednon-reserved
PRIORnon-reserved
PRIVILEGESnon-reserved
PROCEDUREreserved
PTFnon-reserved
PUBLICnon-reserved
PURGEnon-reservednon-reserved
RANGEnon-reservedreserved
RANKreserved
READnon-reserved
READSreserved
REALreserved
RECORDREADERnon-reservednon-reserved
RECORDWRITERnon-reservednon-reserved
RECURSIVEreserved
RECOVERnon-reservednon-reserved
REDUCEnon-reservednon-reserved
REFreserved
REFERENCESreservedreserved
REFERENCINGreserved
REFRESHnon-reservednon-reserved
REGR_AVGXreserved
REGR_AVGYreserved
REGR_COUNTreserved
REGR_INTERCEPTreserved
REGR_R2reserved
REGR_SLOPEreserved
REGR_SXXreserved
REGR_SXYreserved
REGR_SYYreserved
RELATIVEnon-reserved
RELEASEreserved
RENAMEnon-reservednon-reserved
REPAIRnon-reservednon-reserved
REPEATreserved
REPLACEnon-reservednon-reserved
RESETnon-reservednon-reserved
RESIGNALreserved
RESTRICTnon-reservednon-reserved
RESULTreserved
RETURNreserved
RETURNSreserved
REVOKEnon-reservedreserved
RIGHTreservedreserved
RLIKEnon-reservednon-reserved
ROLEnon-reservednon-reserved
ROLESnon-reservednon-reserved
ROLLBACKnon-reservedreserved
ROLLUPnon-reservedreserved
ROUTINEnon-reserved
ROWnon-reservedreserved
ROW_NUMBERreserved
ROWSnon-reservedreserved
RUNNINGnon-reserved
SAVEPOINTreserved
SCHEMAnon-reserved
SCOPEreserved
SCROLLreserved
SEARCHreserved
SECONDreserved
SECTIONnon-reserved
SEEKnon-reserved
SELECTreservedreserved
SENSITIVEreserved
SEPARATEDnon-reservednon-reserved
SERDEnon-reservednon-reserved
SERDEPROPERTIESnon-reservednon-reserved
SESSIONnon-reserved
SESSION_USERreservedreserved
SETnon-reservedreserved
SETSnon-reservednon-reserved
SHOWnon-reservednon-reserved
SIGNALreserved
SIMILARreserved
SINnon-reserved
SINHnon-reserved
SIZEnon-reserved
SKIPnon-reserved
SKEWEDnon-reservednon-reserved
SMALLINTreserved
SOMEreservedreserved
SORTnon-reservednon-reserved
SORTEDnon-reservednon-reserved
SPACEnon-reserved
SPECIFICreserved
SPECIFICTYPEreserved
SQLreserved
SQLCODEnon-reserved
SQLERRORnon-reserved
SQLEXCEPTIONreserved
SQLSTATEreserved
SQLWARNINGreserved
SQRTreserved
STARTnon-reservedreserved
STATEnon-reserved
STATICreserved
STATISTICSnon-reservednon-reserved
STDDEV_POPreserved
STDDEV_SAMPreserved
STOREDnon-reservednon-reserved
STRATIFYnon-reservednon-reserved
STRUCTnon-reservednon-reserved
SUBMULTISETreserved
SUBSETnon-reserved
SUBSTRINGreserved
SUBSTRING_REGEXreserved
SUCCEEDSreserved
SUMreserved
SYMMETRICreserved
SYSTEMreserved
SYSTEM_TIMEreserved
SYSTEM_USERreserved
TABLEreservedreserved
TABLESnon-reservednon-reserved
TABLESAMPLEnon-reservedreserved
TANnon-reserved
TANHnon-reserved
TBLPROPERTIESnon-reservednon-reserved
TEMPORARYnon-reservednon-reserved
TERMINATEDnon-reservednon-reserved
THENreservedreserved
TIMEreserved
TIMESTAMPreserved
TIMEZONE_HOURreserved
TIMEZONE_MINUTEreserved
TOreservedreserved
TOUCHnon-reservednon-reserved
TRAILINGreservedreserved
TRANSACTIONnon-reservednon-reserved
TRANSACTIONSnon-reservednon-reserved
TRANSFORMnon-reservednon-reserved
TRANSLATEreserved
TRANSLATE_REGEXreserved
TRANSLATIONreserved
TREATreserved
TRIGGERreserved
TRIMreserved
TRIM_ARRAYreserved
TRUEnon-reservedreserved
TRUNCATEnon-reservedreserved
UESCAPEreserved
UNARCHIVEnon-reservednon-reserved
UNBOUNDEDnon-reservednon-reserved
UNCACHEnon-reservednon-reserved
UNDERnon-reserved
UNDOreserved
UNIONreservedreserved
UNIQUEreservedreserved
UNKNOWNreserved
UNLOCKnon-reservednon-reserved
UNNESTreserved
UNSETnon-reservednon-reserved
UNTILreserved
UPDATEreserved
UPPERreserved
USAGEnon-reserved
USEnon-reservednon-reserved
USERreservedreserved
USINGreservedreserved
VALUEreserved
VALUESnon-reservedreserved
VALUE_OFreserved
VAR_POPreserved
VAR_SAMPreserved
VARBINARYreserved
VARCHARreserved
VARYINGreservedreserved
VERSIONINGreserved
VIEWnon-reservednon-reserved
WHENreservedreserved
WHENEVERreserved
WHEREreservedreserved
WHILEreserved
WIDTH_BUCKETreserved
WINDOWnon-reservedreserved
WITHreservedreserved
WITHINreserved
WITHOUTreserved
WORKnon-reserved
WRITEnon-reserved
YEARreserved
ZONEnon-reserved
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 8b7b10788df8b..49d5c2282cd70 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -44,6 +44,11 @@ grammar SqlBase; return true; } } + + /** + * When true, ANSI SQL parsing mode is enabled. + */ + public boolean ansi = false; } singleStatement @@ -746,43 +751,35 @@ number ; reserved - : ALL | ANY + // A list of reserved keywords in SparkSQL. These keywords are reserved in all the ANSI SQL standards + // (SQL-92, SQL-99, SQL-2003, SQL-2008, SQL-2011, and SQL-2016) and PostgreSQL. + : ALL | AND | ANY | AS | AUTHORIZATION | BOTH | CASE | CAST | CHECK | COLLATE | COLUMN | CONSTRAINT | CREATE + | CROSS | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DISTINCT | ELSE | END | EXCEPT + | FALSE | FETCH | FOR | FOREIGN | FROM | FULL | GRANT | GROUP | HAVING | IN | INNER | INTERSECT | INTO | IS + | JOIN | LEADING | LEFT | NATURAL | NOT | NULL | ON | ONLY | OR | ORDER | OUTER | OVERLAPS | PRIMARY + | REFERENCES | RIGHT | SELECT | SESSION_USER | SOME | TABLE | THEN | TO | TRAILING | UNION | UNIQUE | USER + | USING | VARYING | WHEN | WHERE | WITH ; nonReserved - : SHOW | TABLES | COLUMNS | COLUMN | PARTITIONS | FUNCTIONS | DATABASES - | ADD - | OVER | PARTITION | RANGE | ROWS | PRECEDING | FOLLOWING | CURRENT | ROW | LAST | FIRST | AFTER - | MAP | ARRAY | STRUCT - | PIVOT | LATERAL | WINDOW | REDUCE | TRANSFORM | SERDE | SERDEPROPERTIES | RECORDREADER - | DELIMITED | FIELDS | TERMINATED | COLLECTION | ITEMS | KEYS | ESCAPED | LINES | SEPARATED - | EXTENDED | REFRESH | CLEAR | CACHE | UNCACHE | LAZY | GLOBAL | TEMPORARY | OPTIONS - | GROUPING | CUBE | ROLLUP - | EXPLAIN | FORMAT | LOGICAL | FORMATTED | CODEGEN | COST - | TABLESAMPLE | USE | TO | BUCKET | PERCENTLIT | OUT | OF - | SET | RESET - | VIEW | REPLACE - | IF - | POSITION - | EXTRACT - | NO | DATA - | START | TRANSACTION | COMMIT | ROLLBACK | IGNORE - | SORT | CLUSTER | DISTRIBUTE | UNSET | TBLPROPERTIES | SKEWED | STORED | DIRECTORIES | LOCATION - | EXCHANGE | ARCHIVE | UNARCHIVE | FILEFORMAT | TOUCH | COMPACT | CONCATENATE | CHANGE - | CASCADE | RESTRICT | BUCKETS | CLUSTERED | SORTED | PURGE | INPUTFORMAT | OUTPUTFORMAT - | DBPROPERTIES | DFS | TRUNCATE | COMPUTE | LIST - | STATISTICS | ANALYZE | PARTITIONED | EXTERNAL | DEFINED | RECORDWRITER - | REVOKE | GRANT | LOCK | UNLOCK | MSCK | REPAIR | RECOVER | EXPORT | IMPORT | LOAD | VALUES | COMMENT | ROLE - | ROLES | COMPACTIONS | PRINCIPALS | TRANSACTIONS | INDEX | INDEXES | LOCKS | OPTION | LOCAL | INPATH - | ASC | DESC | LIMIT | RENAME | SETS - | AT | NULLS | OVERWRITE | ANY | ALTER | AS | BETWEEN | BY | CREATE | DELETE - | DESCRIBE | DROP | EXISTS | FALSE | FOR | GROUP | IN | INSERT | INTO | IS |LIKE - | NULL | ORDER | OUTER | TABLE | TRUE | WITH | RLIKE - | AND | CASE | CAST | DISTINCT | DIV | ELSE | END | FUNCTION | INTERVAL | MACRO | OR | STRATIFY | THEN - | UNBOUNDED | WHEN - | DATABASE | SELECT | FROM | WHERE | HAVING | TO | TABLE | WITH | NOT - | DIRECTORY - | BOTH | LEADING | TRAILING + : ADD | AFTER | ALL | ALTER | ANALYZE | AND | ANY | ARCHIVE | ARRAY | AS | ASC | AT | BETWEEN | BOTH + | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN + | COLLECTION | COLUMN | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST + | CREATE | CUBE | CURRENT | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED + | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY | DISTINCT | DISTRIBUTE | DIV | DROP | ELSE | END + | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT | EXTENDED | EXTERNAL | EXTRACT | FALSE | FIELDS + | FILEFORMAT | FIRST | FOLLOWING | FOR | FORMAT | FORMATTED | FROM | FUNCTION | FUNCTIONS | GLOBAL + | GRANT | GROUP | GROUPING | HAVING | IF | IGNORE | IMPORT | IN | INDEX | INDEXES | INPATH | INPUTFORMAT + | INSERT | INTERVAL | INTO | IS |LIKE | ITEMS | KEYS | LAST | LATERAL | LAZY | LEADING | LIMIT | LINES + | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO | MAP | MSCK | NO | NOT | NULL | NULLS + | OF | OPTION | OPTIONS | OR | ORDER | OUT | OUTER | OUTPUTFORMAT | OVER | OVERWRITE | PARTITION + | PARTITIONED | PARTITIONS | PERCENTLIT | PIVOT | POSITION | PRECEDING | PRINCIPALS | PURGE | RANGE + | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFRESH | RENAME | REPAIR | REPLACE | RESET | RESTRICT + | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SELECT | SEPARATED | SERDE + | SERDEPROPERTIES | SET | SETS | SHOW | SKEWED | SORT | SORTED | START | STATISTICS | STORED | STRATIFY + | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | THEN | TO | TOUCH + | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE + | UNLOCK | UNSET | USE | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; SELECT: 'SELECT'; @@ -1019,6 +1016,25 @@ OPTION: 'OPTION'; ANTI: 'ANTI'; LOCAL: 'LOCAL'; INPATH: 'INPATH'; +AUTHORIZATION: 'AUTHORIZATION'; +CHECK: 'CHECK'; +COLLATE: 'COLLATE'; +CONSTRAINT: 'CONSTRAINT'; +CURRENT_DATE: 'CURRENT_DATE'; +CURRENT_TIME: 'CURRENT_TIME'; +CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; +CURRENT_USER: 'CURRENT_USER'; +FETCH: 'FETCH'; +FOREIGN: 'FOREIGN'; +ONLY: 'ONLY'; +OVERLAPS: 'OVERLAPS'; +PRIMARY: 'PRIMARY'; +REFERENCES: 'REFERENCES'; +SESSION_USER: 'SESSION_USER'; +SOME: 'SOME'; +UNIQUE: 'UNIQUE'; +USER: 'USER'; +VARYING: 'VARYING'; STRING : '\'' ( ~('\''|'\\') | ('\\' .) )* '\'' diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index e1cc7412a08c7..e4878c87d920e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -51,6 +51,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging def this() = this(new SQLConf()) + def ansi: Boolean = conf.ansiParserEnabled + protected def typedVisit[T](ctx: ParseTree): T = { ctx.accept(this).asInstanceOf[T] } @@ -1407,11 +1409,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitIdentifier(ctx: IdentifierContext): String = withOrigin(ctx) { val keyword = ctx.getText if (ctx.reserved() != null) { - // This option is expected to come from `spark.sql.parser.ansi.enabled` (pr20433): - // val ansi = conf.ansiParserEnabled - val ansi = true if (ansi) { - throw new ParseException(s"'$keyword' is reserved in the ANSI SQL-2011 standard.", ctx) + throw new ParseException( + s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) } } keyword diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala index 7d8cb1f18b4b5..1d5de0a353304 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala @@ -85,6 +85,7 @@ abstract class AbstractSqlParser extends ParserInterface with Logging { lexer.removeErrorListeners() lexer.addErrorListener(ParseErrorListener) lexer.legacy_setops_precedence_enbled = SQLConf.get.setOpsPrecedenceEnforced + lexer.ansi = SQLConf.get.ansiParserEnabled val tokenStream = new CommonTokenStream(lexer) val parser = new SqlBaseParser(tokenStream) @@ -92,6 +93,7 @@ abstract class AbstractSqlParser extends ParserInterface with Logging { parser.removeErrorListeners() parser.addErrorListener(ParseErrorListener) parser.legacy_setops_precedence_enbled = SQLConf.get.setOpsPrecedenceEnforced + parser.ansi = SQLConf.get.ansiParserEnabled try { try { @@ -131,7 +133,7 @@ class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser { /** For test-only. */ object CatalystSqlParser extends AbstractSqlParser { - val astBuilder = new AstBuilder(new SQLConf()) + val astBuilder = new AstBuilder(SQLConf.get) } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index d285e007dac1b..783f33384cc6a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -314,6 +314,12 @@ object SQLConf { .booleanConf .createWithDefault(true) + val ANSI_SQL_PARSER = + buildConf("spark.sql.parser.ansi.enabled") + .doc("When true, tries to conform to ANSI SQL syntax.") + .booleanConf + .createWithDefault(false) + val ESCAPED_STRING_LITERALS = buildConf("spark.sql.parser.escapedStringLiterals") .internal() .doc("When true, string literals (including regex patterns) remain escaped in our SQL " + @@ -1832,6 +1838,8 @@ class SQLConf extends Serializable with Logging { def constraintPropagationEnabled: Boolean = getConf(CONSTRAINT_PROPAGATION_ENABLED) + def ansiParserEnabled: Boolean = getConf(ANSI_SQL_PARSER) + def escapedStringLiterals: Boolean = getConf(ESCAPED_STRING_LITERALS) def fileCompressionFactor: Double = getConf(FILE_COMRESSION_FACTOR) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index 4ea2b1b9ebcc9..cb8ab61f47ebd 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -18,12 +18,14 @@ package org.apache.spark.sql.catalyst.parser import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.plans.SQLHelper +import org.apache.spark.sql.internal.SQLConf -class TableIdentifierParserSuite extends SparkFunSuite { +class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { import CatalystSqlParser._ // Add "$elem$", "$value$" & "$key$" - val hiveNonReservedKeyword = Array("add", "admin", "after", "analyze", "archive", "asc", "before", + val hiveNonReservedKeywords = Seq("add", "admin", "after", "analyze", "archive", "asc", "before", "bucket", "buckets", "cascade", "change", "cluster", "clustered", "clusterstatus", "collection", "columns", "comment", "compact", "compactions", "compute", "concatenate", "continue", "cost", "data", "day", "databases", "datetime", "dbproperties", "deferred", "defined", "delimited", @@ -42,7 +44,7 @@ class TableIdentifierParserSuite extends SparkFunSuite { "tblproperties", "temporary", "terminated", "tinyint", "touch", "transactions", "unarchive", "undo", "uniontype", "unlock", "unset", "unsigned", "uri", "use", "utc", "utctimestamp", "view", "while", "year", "work", "transaction", "write", "isolation", "level", "snapshot", - "autocommit", "alter", "array", "as", "authorization", "between", "bigint", + "autocommit", "all", "any", "alter", "array", "as", "authorization", "between", "bigint", "binary", "boolean", "both", "by", "create", "cube", "current_date", "current_timestamp", "cursor", "date", "decimal", "delete", "describe", "double", "drop", "exists", "external", "false", "fetch", "float", "for", "grant", "group", "grouping", "import", "in", @@ -53,10 +55,19 @@ class TableIdentifierParserSuite extends SparkFunSuite { "bigint", "binary", "boolean", "current_date", "current_timestamp", "date", "double", "float", "int", "smallint", "timestamp", "at", "position", "both", "leading", "trailing", "extract") - val hiveStrictNonReservedKeyword = Seq("anti", "full", "inner", "left", "semi", "right", + val hiveStrictNonReservedKeywords = Seq("anti", "full", "inner", "left", "semi", "right", "natural", "union", "intersect", "except", "database", "on", "join", "cross", "select", "from", "where", "having", "from", "to", "table", "with", "not") + val sparkSQLReservedKeywords = Seq("all", "and", "any", "as", "authorization", "both", "case", + "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", + "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", + "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", + "intersect", "into", "is", "join", "leading", "left", "natural", "not", "null", "on", "only", + "or", "order", "outer", "overlaps", "primary", "references", "right", "select", "session_user", + "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "varying", + "when", "where", "with") + test("table identifier") { // Regular names. assert(TableIdentifier("q") === parseTableIdentifier("q")) @@ -76,18 +87,31 @@ class TableIdentifierParserSuite extends SparkFunSuite { assert(TableIdentifier("x.y.z", None) === parseTableIdentifier("`x.y.z`")) } - test("table identifier - reserved by the SQL-2011 standard") { - Seq("all", "any").foreach { keyword => - val errMsg = intercept[ParseException] { parseTableIdentifier(keyword) }.getMessage - assert(errMsg.contains(s"'$keyword' is reserved in the ANSI SQL-2011 standard.")) - assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) - assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) + test("table identifier - reserved keywords") { + withSQLConf(SQLConf.ANSI_SQL_PARSER.key -> "true") { + sparkSQLReservedKeywords.foreach { keyword => + val errMsg = intercept[ParseException] { + parseTableIdentifier(keyword) + }.getMessage + assert(errMsg.contains(s"'$keyword' is reserved and you cannot use this keyword " + + "as an identifier.")) + assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) + assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) + } + } + + // If an ANSI mode disabled, we can use these keywords as identifiers + withSQLConf(SQLConf.ANSI_SQL_PARSER.key -> "false") { + sparkSQLReservedKeywords.foreach { keyword => + assert(TableIdentifier(keyword) === parseTableIdentifier(s"$keyword")) + assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.$keyword")) + } } } test("table identifier - strict keywords") { // SQL Keywords. - hiveStrictNonReservedKeyword.foreach { keyword => + hiveStrictNonReservedKeywords.foreach { keyword => assert(TableIdentifier(keyword) === parseTableIdentifier(keyword)) assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) @@ -96,7 +120,7 @@ class TableIdentifierParserSuite extends SparkFunSuite { test("table identifier - non reserved keywords") { // Hive keywords are allowed. - hiveNonReservedKeyword.foreach { nonReserved => + hiveNonReservedKeywords.foreach { nonReserved => assert(TableIdentifier(nonReserved) === parseTableIdentifier(nonReserved)) } } From 05cf038a6d43850d114e3ad48c1ee86ad5621ef7 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Fri, 1 Feb 2019 15:40:24 +0900 Subject: [PATCH 03/18] Fix --- ...sql-reserved-and-non-reserved-key-words.md | 654 +++++++++--------- .../spark/sql/catalyst/parser/SqlBase.g4 | 63 +- .../sql/catalyst/parser/AstBuilder.scala | 8 +- .../parser/TableIdentifierParserSuite.scala | 75 +- 4 files changed, 437 insertions(+), 363 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index 23b2cdcad3021..d54961b98e079 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -10,78 +10,78 @@ A full list of reserved/non-reserved key words is as follows: - - - - + + + + - + - + - - + + - - non-reserved - - + + non-reserved + + - - + + - - - - - + + + + + - - - - - - + + + + + + - + - - - + + + - + - - - + + + - - - - + + + + - + - - + + - + - - + + @@ -91,102 +91,100 @@ A full list of reserved/non-reserved key words is as follows: - - - + + + - - - - - - - - - - + + + + + + + + + + - - - + + + - + - + - - - - - + + + + + - + - - + + - - + + - - - - - - - - - + + + + + + + + + - - - + + + - - + + - + - + - - - + + + - - - + + + - - + + - - - - - - + + + + - + - + - - + + - - + + @@ -196,235 +194,235 @@ A full list of reserved/non-reserved key words is as follows: - + - - + + - - - + + + - - - + + + - - + + - - + + - - - + + + - + - + - - + + - + - + - + - - + + - + - + - + - - - - - - - - - - + + + + + + + + + + - - - + + + - + - + - + - + - + - - + + - - + + - + - - + + - - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - + + + - - - - + + + + - - + + - - + + - - - - - + + + + + - - - + + + - + - + - + - + - + - - - + + + - - - - - - - + + + + + + + - + - + - - - + + + - - - + + + - - - - - + + + + + - - - - + + + + - + - + - + - - - - - - - - - - - + + + + + + + + + + + - + - + - - - + + + @@ -432,137 +430,137 @@ A full list of reserved/non-reserved key words is as follows: - + - + - - - - - - - - - + + + + + + + + + - + - + - - - - - - + + + + + + - + - - - - - - - - - - + + + + + + + + + + - - + + - - + + - - - - - - - - - - + + + + + + + + + + - - + + - - - - + + + + - - - - - - - + + + + + + + - + - - + + - + - + - - - - + + + + - + - - - - - + + + + + - + - + - - + + - - - - - - + + + + + +
Key WordSpark SQLSQL-2011
ABSreserved
ABSOLUTEnon-reserved
ACOSnon-reserved
ACTIONnon-reserved
ABSnon-reservedreserved
ABSOLUTEnon-reservednon-reserved
ACOSnon-reservednon-reserved
ACTIONnon-reservednon-reserved
ADDnon-reservednon-reserved
AFTERnon-reservednon-reserved
ALLreservedreserved
ALLOCATEreserved
ALLOCATEnon-reservedreserved
ALTERnon-reservedreserved
ANALYZEnon-reservednon-reserved
ANDreservedreserved
ANYreservedreserved
AREreserved
AREnon-reservedreserved
ARCHIVEnon-reservednon-reserved
ARRAYnon-reservedreserved
ARRAY_AGGreserved
ARRAY_MAX_CARDINALITYreserved
ARRAY_AGGnon-reservedreserved
ARRAY_MAX_CARDINALITYnon-reservedreserved
ASreservedreserved
ASCnon-reservednon-reserved
ASENSITIVEreserved
ASINreserved
ASSERTIONnon-reserved
ASYMMETRICreserved
ASENSITIVEnon-reservedreserved
ASINnon-reservedreserved
ASSERTIONnon-reservednon-reserved
ASYMMETRICnon-reservedreserved
ATnon-reservedreserved
ATANnon-reserved
ATOMICreserved
ATANnon-reservednon-reserved
ATOMICnon-reservedreserved
AUTHORIZATIONreservedreserved
AVGreserved
BEFOREnon-reserved
BEGINreserved
BEGIN_FRAMEreserved
BEGIN_PARTITIONreserved
AVGnon-reservedreserved
BEFOREnon-reservednon-reserved
BEGINnon-reservedreserved
BEGIN_FRAMEnon-reservedreserved
BEGIN_PARTITIONnon-reservedreserved
BETWEENnon-reservedreserved
BIGINTreserved
BINARYreserved
BITnon-reserved
BIT_LENGTHnon-reserved
BLOBreserved
BOOLEANreserved
BIGINTnon-reservedreserved
BINARYnon-reservedreserved
BITnon-reservednon-reserved
BIT_LENGTHnon-reservednon-reserved
BLOBnon-reservedreserved
BOOLEANnon-reservedreserved
BOTHreservedreserved
BREADTHnon-reserved
BREADTHnon-reservednon-reserved
BUCKETnon-reservednon-reserved
BUCKETSnon-reservednon-reserved
BYnon-reservedreserved
CACHEnon-reservednon-reserved
CALLreserved
CALLEDreserved
CARDINALITYreserved
CALLnon-reservedreserved
CALLEDnon-reservedreserved
CARDINALITYnon-reservedreserved
CASCADEnon-reservedreserved
CASCADEDreserved
CASCADEDnon-reservedreserved
CASEreservedreserved
CASTreservedreserved
CATALOGnon-reserved
CEILreserved
CEILINGreserved
CATALOGnon-reservednon-reserved
CEILnon-reservedreserved
CEILINGnon-reservedreserved
CHANGEnon-reservednon-reserved
CHARreserved
CHAR_LENGTHreserved
CHARACTERreserved
CHARACTER_LENGTHreserved
CHARnon-reservedreserved
CHAR_LENGTHnon-reservedreserved
CHARACTERnon-reservedreserved
CHARACTER_LENGTHnon-reservedreserved
CHECKreservedreserved
CLASSIFIERnon-reserved
CLASSIFIERnon-reservednon-reserved
CLEARnon-reservednon-reserved
CLOBreserved
CLOSEreserved
CLOBnon-reservedreserved
CLOSEnon-reservedreserved
CLUSTERnon-reservednon-reserved
CLUSTEREDnon-reservednon-reserved
COALESCEreserved
COALESCEnon-reservedreserved
CODEGENnon-reservednon-reserved
COLLATEreservedreserved
COLLATIONnon-reserved
COLLECTreserved
COLLATIONnon-reservednon-reserved
COLLECTnon-reservedreserved
COLLECTIONnon-reservednon-reserved
COLUMNreservedreserved
COLUMNSnon-reservednon-reserved
COMPACTIONSnon-reservednon-reserved
COMPUTEnon-reservednon-reserved
CONCATENATEnon-reservednon-reserved
CONDITIONreserved
CONNECTnon-reserved
CONNECTIONnon-reserved
CONDITIONnon-reservedreserved
CONNECTnon-reservednon-reserved
CONNECTIONnon-reservednon-reserved
CONSTRAINTreservedreserved
CONSTRAINTSnon-reserved
CONSTRUCTORnon-reserved
CONTAINSnon-reserved
CONTINUEnon-reserved
CONVERTreserved
COPYnon-reserved
CORRreserved
CORRESPONDINGreserved
COSnon-reserved
COSHnon-reserved
CONSTRAINTSnon-reservednon-reserved
CONSTRUCTORnon-reservednon-reserved
CONTAINSnon-reservednon-reserved
CONTINUEnon-reservednon-reserved
CONVERTnon-reservedreserved
COPYnon-reservednon-reserved
CORRnon-reservedreserved
CORRESPONDINGnon-reservedreserved
COSnon-reservednon-reserved
COSHnon-reservednon-reserved
COSTnon-reservednon-reserved
COUNTreserved
COVAR_POPreserved
COVAR_SAMPreserved
COUNTnon-reservedreserved
COVAR_POPnon-reservedreserved
COVAR_SAMPnon-reservedreserved
CREATEreservedreserved
CROSSreservedreserved
CUBEnon-reservedreserved
CUME_DISTreserved
CUME_DISTnon-reservedreserved
CURRENTnon-reservedreserved
CURRENT_CATALOGreserved
CURRENT_CATALOGnon-reservedreserved
CURRENT_DATEreservedreserved
CURRENT_DEFAULT_TRANSFORM_GROUPreserved
CURRENT_PATHreserved
CURRENT_ROLEreserved
CURRENT_ROWreserved
CURRENT_SCHEMAreserved
CURRENT_DEFAULT_TRANSFORM_GROUPnon-reservedreserved
CURRENT_PATHnon-reservedreserved
CURRENT_ROLEnon-reservedreserved
CURRENT_ROWnon-reservedreserved
CURRENT_SCHEMAnon-reservedreserved
CURRENT_TIMEreservedreserved
CURRENT_TIMESTAMPreservedreserved
CURRENT_TRANSFORM_GROUP_FOR_TYPEreserved
CURRENT_TRANSFORM_GROUP_FOR_TYPEnon-reservedreserved
CURRENT_USERreservedreserved
CURSORreserved
CYCLEreserved
CURSORnon-reservedreserved
CYCLEnon-reservedreserved
DATAnon-reservednon-reserved
DATABASEnon-reservednon-reserved
DATABASESnon-reservednon-reserved
DATEreserved
DAYreserved
DATEnon-reservedreserved
DAYnon-reservedreserved
DBPROPERTIESnon-reservednon-reserved
DEALLOCATEreserved
DECreserved
DECFLOATnon-reserved
DECIMALreserved
DECLAREreserved
DEFAULTreserved
DEFERRABLEnon-reserved
DEFERREDnon-reserved
DEFINEnon-reserved
DEALLOCATEnon-reservedreserved
DECnon-reservedreserved
DECFLOATnon-reservednon-reserved
DECIMALnon-reservedreserved
DECLAREnon-reservedreserved
DEFAULTnon-reservedreserved
DEFERRABLEnon-reservednon-reserved
DEFERREDnon-reservednon-reserved
DEFINEnon-reservednon-reserved
DEFINEDnon-reservednon-reserved
DELETEnon-reservedreserved
DELIMITEDnon-reservednon-reserved
DENSE_RANKreserved
DEPTHnon-reserved
DEREFreserved
DENSE_RANKnon-reservedreserved
DEPTHnon-reservednon-reserved
DEREFnon-reservedreserved
DESCnon-reservednon-reserved
DESCRIBEnon-reservedreserved
DESCRIPTORnon-reserved
DETERMINISTICreserved
DESCRIPTORnon-reservednon-reserved
DETERMINISTICnon-reservedreserved
DFSnon-reservednon-reserved
DIAGNOSTICSnon-reserved
DIAGNOSTICSnon-reservednon-reserved
DIRECTORIESnon-reservednon-reserved
DIRECTORYnon-reservednon-reserved
DISCONNECTreserved
DISCONNECTnon-reservedreserved
DISTINCTreservedreserved
DISTRIBUTEnon-reservednon-reserved
DIVnon-reservednon-reserved
DOreserved
DOMAINnon-reserved
DOUBLEreserved
DOnon-reservedreserved
DOMAINnon-reservednon-reserved
DOUBLEnon-reservedreserved
DROPnon-reservedreserved
DYNAMICreserved
EACHreserved
ELEMENTreserved
DYNAMICnon-reservedreserved
EACHnon-reservedreserved
ELEMENTnon-reservedreserved
ELSEreservedreserved
ELSEIFreserved
EMPTYnon-reserved
ELSEIFnon-reservedreserved
EMPTYnon-reservednon-reserved
ENDreservedreserved
END_FRAMEreserved
END_PARTITIONreserved
ENDreserved
EXECreserved
EQUALSnon-reserved
ESCAPEreserved
END_FRAMEnon-reservedreserved
END_PARTITIONnon-reservedreserved
EQUALSnon-reservednon-reserved
ESCAPEnon-reservedreserved
ESCAPEDnon-reservednon-reserved
EVERYreserved
EVERYnon-reservedreserved
EXCEPTreservedreserved
EXCEPTIONnon-reserved
EXCEPTIONnon-reservednon-reserved
EXCHANGEnon-reservednon-reserved
EXECreserved
EXECUTEreserved
EXECnon-reservedreserved
EXECUTEnon-reservedreserved
EXISTSnon-reservedreserved
EXITnon-reserved
EXPnon-reserved
EXITnon-reservednon-reserved
EXPnon-reservednon-reserved
EXPLAINnon-reservednon-reserved
EXPORTnon-reservednon-reserved
EXTENDEDnon-reservednon-reserved
FETCHreservedreserved
FIELDSnon-reservednon-reserved
FILEFORMATnon-reservednon-reserved
FILTERreserved
FILTERnon-reservedreserved
FIRSTnon-reservednon-reserved
FIRST_VALUEreserved
FLOATreserved
FIRST_VALUEnon-reservedreserved
FLOATnon-reservedreserved
FOLLOWINGnon-reservednon-reserved
FORreservedreserved
FOREIGNreservedreserved
FORMATnon-reservednon-reserved
FORMATTEDnon-reservednon-reserved
FOUNDnon-reserved
FRAME_ROWreserved
FREEreserved
FOUNDnon-reservednon-reserved
FRAME_ROWnon-reservedreserved
FREEnon-reservedreserved
FROMreservedreserved
FULLreservedreserved
FUNCTIONnon-reservedreserved
FUNCTIONSnon-reservednon-reserved
FUSIONnon-reserved
GENERALnon-reserved
GETreserved
FUSIONnon-reservednon-reserved
GENERALnon-reservednon-reserved
GETnon-reservedreserved
GLOBALnon-reservedreserved
GOnon-reserved
GOTOnon-reserved
GOnon-reservednon-reserved
GOTOnon-reservednon-reserved
GRANTreservedreserved
GROUPreservedreserved
GROUPINGnon-reservedreserved
GROUPSreserved
HANDLERreserved
GROUPSnon-reservedreserved
HANDLERnon-reservedreserved
HAVINGreservedreserved
HOLDreserved
HOURreserved
IDENTITYreserved
HOLDnon-reservedreserved
HOURnon-reservedreserved
IDENTITYnon-reservedreserved
IFnon-reservedreserved
IGNOREnon-reservednon-reserved
IMMEDIATEnon-reserved
IMMEDIATEnon-reservednon-reserved
IMPORTnon-reservednon-reserved
INreservedreserved
INDICATORreserved
INDICATORnon-reservedreserved
INDEXnon-reservednon-reserved
INDEXESnon-reservednon-reserved
INITIALnon-reserved
INITIALLYnon-reserved
INITIALnon-reservednon-reserved
INITIALLYnon-reservednon-reserved
INNERreservedreserved
INOUTreserved
INOUTnon-reservedreserved
INPATHnon-reservednon-reserved
INPUTnon-reserved
INPUTnon-reservednon-reserved
INPUTFORMATnon-reservednon-reserved
INSENSITIVEreserved
INSENSITIVEnon-reservedreserved
INSERTnon-reservedreserved
INTreserved
INTEGERreserved
INTnon-reservedreserved
INTEGERnon-reservedreserved
INTERSECTreservedreserved
INTERSECTIONreserved
INTERSECTIONnon-reservedreserved
INTERVALnon-reservedreserved
INTOreservedreserved
ISreservedreserved
ISOLATIONnon-reserved
ISOLATIONnon-reservednon-reserved
ITEMSnon-reservednon-reserved
ITERATEreserved
ITERATEnon-reservedreserved
JOINreservedreserved
JSON_ARRAYnon-reserved
JSON_ARRAYAGGnon-reserved
JSON_EXISTSnon-reserved
JSON_OBJECTnon-reserved
JSON_OBJECTAGGnon-reserved
JSON_QUERYnon-reserved
JSON_TABLEnon-reserved
JSON_TABLE_PRIMITIVEnon-reserved
JSON_VALUEnon-reserved
KEYnon-reserved
JSON_ARRAYnon-reservednon-reserved
JSON_ARRAYAGGnon-reservednon-reserved
JSON_EXISTSnon-reservednon-reserved
JSON_OBJECTnon-reservednon-reserved
JSON_OBJECTAGGnon-reservednon-reserved
JSON_QUERYnon-reservednon-reserved
JSON_TABLEnon-reservednon-reserved
JSON_TABLE_PRIMITIVEnon-reservednon-reserved
JSON_VALUEnon-reservednon-reserved
KEYnon-reservednon-reserved
KEYSnon-reservednon-reserved
LAGnon-reserved
LANGUAGEreserved
LARGEreserved
LAGnon-reservednon-reserved
LANGUAGEnon-reservedreserved
LARGEnon-reservedreserved
LASTnon-reservednon-reserved
LAST_VALUEreserved
LAST_VALUEnon-reservedreserved
LATERALnon-reservedreserved
LAZYnon-reservednon-reserved
LEADreserved
LEADnon-reservedreserved
LEADINGreservedreserved
LEAVEreserved
LEAVEnon-reservedreserved
LEFTreservedreserved
LEVELnon-reserved
LEVELnon-reservednon-reserved
LIKEnon-reservedreserved
LIKE_REGEXreserved
LIKE_REGEXnon-reservedreserved
LIMITnon-reservednon-reserved
LINESnon-reservednon-reserved
LISTnon-reservednon-reserved
LISTAGGnon-reserved
LNreserved
LISTAGGnon-reservednon-reserved
LNnon-reservedreserved
LOADnon-reservednon-reserved
LOCALnon-reservedreserved
LOCALTIMEreserved
LOCALTIMESTAMPreserved
LOCALTIMEnon-reservedreserved
LOCALTIMESTAMPnon-reservedreserved
LOCATIONnon-reservednon-reserved
LOCATORnon-reserved
LOCATORnon-reservednon-reserved
LOCKnon-reservednon-reserved
LOCKSnon-reservednon-reserved
LOGnon-reserved
LOG10non-reserved
LOGnon-reservednon-reserved
LOG10non-reservednon-reserved
LOGICALnon-reservednon-reserved
LOOPreserved
LOWERreserved
LOOPnon-reservedreserved
LOWERnon-reservedreserved
MACROnon-reservednon-reserved
MAPnon-reservednon-reserved
MATCHreserved
MATCH_NUMBERnon-reserved
MATCH_RECOGNIZEnon-reserved
MATCHESnon-reserved
MAXreserved
MEMBERreserved
MERGEreserved
METHODreserved
MINreserved
MINUTEreserved
MODreserved
MODIFIESreserved
MODULEreserved
MONTHreserved
MATCHnon-reservedreserved
MATCH_NUMBERnon-reservednon-reserved
MATCH_RECOGNIZEnon-reservednon-reserved
MATCHESnon-reservednon-reserved
MAXnon-reservedreserved
MEMBERnon-reservedreserved
MERGEnon-reservedreserved
METHODnon-reservedreserved
MINnon-reservedreserved
MINUTEnon-reservedreserved
MODnon-reservedreserved
MODIFIESnon-reservedreserved
MODULEnon-reservedreserved
MONTHnon-reservedreserved
MSCKnon-reservednon-reserved
MULTISETreserved
NAMESnon-reserved
NATIONALreserved
MULTISETnon-reservedreserved
NAMESnon-reservednon-reserved
NATIONALnon-reservedreserved
NATURALreservedreserved
NCHARreserved
NCLOBreserved
NEWreserved
NEXTnon-reserved
NCHARnon-reservedreserved
NCLOBnon-reservedreserved
NEWnon-reservedreserved
NEXTnon-reservednon-reserved
NOnon-reservedreserved
NONEreserved
NORMALIZEreserved
NONEnon-reservedreserved
NORMALIZEnon-reservedreserved
NOTreservedreserved
NTH_VALUEreserved
NTILEreserved
NTH_VALUEnon-reservedreserved
NTILEnon-reservedreserved
NULLreservedreserved
NULLSnon-reservednon-reserved
NULLIFreserved
NUMERICreserved
OBJECTnon-reserved
OCCURRENCES_REGEXnon-reserved
OCTET_LENGTHreserved
NULLIFnon-reservedreserved
NUMERICnon-reservedreserved
OBJECTnon-reservednon-reserved
OCCURRENCES_REGEXnon-reservednon-reserved
OCTET_LENGTHnon-reservedreserved
OFnon-reservedreserved
OFFSETreserved
OLDreserved
OMITnon-reserved
OFFSETnon-reservedreserved
OLDnon-reservedreserved
OMITnon-reservednon-reserved
ONreservedreserved
ONEnon-reserved
ONEnon-reservednon-reserved
ONLYreservedreserved
OPENreserved
OPENnon-reservedreserved
OPTIONnon-reservednon-reserved
OPTIONSnon-reservednon-reserved
ORreservedreserved
ORDERreservedreserved
ORDINALITYnon-reserved
ORDINALITYnon-reservednon-reserved
OUTnon-reservedreserved
OUTERreservedreserved
OUTPUTnon-reserved
OUTPUTnon-reservednon-reserved
OUTPUTFORMATnon-reservednon-reserved
OVERnon-reservednon-reserved
OVERLAPSreservedreserved
OVERLAYreserved
OVERLAYnon-reservedreserved
OVERWRITEnon-reservednon-reserved
PADnon-reserved
PARAMETERreserved
PARTIALnon-reserved
PADnon-reservednon-reserved
PARAMETERnon-reservedreserved
PARTIALnon-reservednon-reserved
PARTITIONnon-reservedreserved
PARTITIONEDnon-reservednon-reserved
PARTITIONSnon-reservednon-reserved
PATHnon-reserved
PATTERNnon-reserved
PERnon-reserved
PERCENTreserved
PERCENT_RANKreserved
PERCENTILE_CONTreserved
PERCENTILE_DISCreserved
PATHnon-reservednon-reserved
PATTERNnon-reservednon-reserved
PERnon-reservednon-reserved
PERCENTnon-reservedreserved
PERCENT_RANKnon-reservedreserved
PERCENTILE_CONTnon-reservedreserved
PERCENTILE_DISCnon-reservedreserved
PERCENTLITnon-reservednon-reserved
PERIODreserved
PERIODnon-reservedreserved
PIVOTnon-reservednon-reserved
PORTIONreserved
PORTIONnon-reservedreserved
POSITIONnon-reservedreserved
POSITION_REGEXreserved
POWERreserved
PRECEDESreserved
POSITION_REGEXnon-reservedreserved
POWERnon-reservedreserved
PRECEDESnon-reservedreserved
PRECEDINGnon-reservednon-reserved
PRECISIONreserved
PREPAREreserved
PRESERVEnon-reserved
PRECISIONnon-reservedreserved
PREPAREnon-reservedreserved
PRESERVEnon-reservednon-reserved
PRIMARYreservedreserved
PRINCIPALSnon-reservednon-reserved
PRIORnon-reserved
PRIVILEGESnon-reserved
PROCEDUREreserved
PTFnon-reserved
PUBLICnon-reserved
PRIORnon-reservednon-reserved
PRIVILEGESnon-reservednon-reserved
PROCEDUREnon-reservedreserved
PTFnon-reservednon-reserved
PUBLICnon-reservednon-reserved
PURGEnon-reservednon-reserved
RANGEnon-reservedreserved
RANKreserved
READnon-reserved
READSreserved
REALreserved
RANKnon-reservedreserved
READnon-reservednon-reserved
READSnon-reservedreserved
REALnon-reservedreserved
RECORDREADERnon-reservednon-reserved
RECORDWRITERnon-reservednon-reserved
RECURSIVEreserved
RECURSIVEnon-reservedreserved
RECOVERnon-reservednon-reserved
REDUCEnon-reservednon-reserved
REFreserved
REFnon-reservedreserved
REFERENCESreservedreserved
REFERENCINGreserved
REFERENCINGnon-reservedreserved
REFRESHnon-reservednon-reserved
REGR_AVGXreserved
REGR_AVGYreserved
REGR_COUNTreserved
REGR_INTERCEPTreserved
REGR_R2reserved
REGR_SLOPEreserved
REGR_SXXreserved
REGR_SXYreserved
REGR_SYYreserved
RELATIVEnon-reserved
RELEASEreserved
REGR_AVGXnon-reservedreserved
REGR_AVGYnon-reservedreserved
REGR_COUNTnon-reservedreserved
REGR_INTERCEPTnon-reservedreserved
REGR_R2non-reservedreserved
REGR_SLOPEnon-reservedreserved
REGR_SXXnon-reservedreserved
REGR_SXYnon-reservedreserved
REGR_SYYnon-reservedreserved
RELATIVEnon-reservednon-reserved
RELEASEnon-reservedreserved
RENAMEnon-reservednon-reserved
REPAIRnon-reservednon-reserved
REPEATreserved
REPEATnon-reservedreserved
REPLACEnon-reservednon-reserved
RESETnon-reservednon-reserved
RESIGNALreserved
RESIGNALnon-reservedreserved
RESTRICTnon-reservednon-reserved
RESULTreserved
RETURNreserved
RETURNSreserved
RESULTnon-reservedreserved
RETURNnon-reservedreserved
RETURNSnon-reservedreserved
REVOKEnon-reservedreserved
RIGHTreservedreserved
RLIKEnon-reservednon-reserved
ROLESnon-reservednon-reserved
ROLLBACKnon-reservedreserved
ROLLUPnon-reservedreserved
ROUTINEnon-reserved
ROUTINEnon-reservednon-reserved
ROWnon-reservedreserved
ROW_NUMBERreserved
ROW_NUMBERnon-reservedreserved
ROWSnon-reservedreserved
RUNNINGnon-reserved
SAVEPOINTreserved
SCHEMAnon-reserved
SCOPEreserved
SCROLLreserved
SEARCHreserved
SECONDreserved
SECTIONnon-reserved
SEEKnon-reserved
RUNNINGnon-reservednon-reserved
SAVEPOINTnon-reservedreserved
SCHEMAnon-reservednon-reserved
SCOPEnon-reservedreserved
SCROLLnon-reservedreserved
SEARCHnon-reservedreserved
SECONDnon-reservedreserved
SECTIONnon-reservednon-reserved
SEEKnon-reservednon-reserved
SELECTreservedreserved
SENSITIVEreserved
SENSITIVEnon-reservedreserved
SEPARATEDnon-reservednon-reserved
SERDEnon-reservednon-reserved
SERDEPROPERTIESnon-reservednon-reserved
SESSIONnon-reserved
SESSIONnon-reservednon-reserved
SESSION_USERreservedreserved
SETnon-reservedreserved
SETSnon-reservednon-reserved
SHOWnon-reservednon-reserved
SIGNALreserved
SIMILARreserved
SINnon-reserved
SINHnon-reserved
SIZEnon-reserved
SKIPnon-reserved
SIGNALnon-reservedreserved
SIMILARnon-reservedreserved
SINnon-reservednon-reserved
SINHnon-reservednon-reserved
SIZEnon-reservednon-reserved
SKIPnon-reservednon-reserved
SKEWEDnon-reservednon-reserved
SMALLINTreserved
SMALLINTnon-reservedreserved
SOMEreservedreserved
SORTnon-reservednon-reserved
SORTEDnon-reservednon-reserved
SPACEnon-reserved
SPECIFICreserved
SPECIFICTYPEreserved
SQLreserved
SQLCODEnon-reserved
SQLERRORnon-reserved
SQLEXCEPTIONreserved
SQLSTATEreserved
SQLWARNINGreserved
SQRTreserved
SPACEnon-reservednon-reserved
SPECIFICnon-reservedreserved
SPECIFICTYPEnon-reservedreserved
SQLnon-reservedreserved
SQLCODEnon-reservednon-reserved
SQLERRORnon-reservednon-reserved
SQLEXCEPTIONnon-reservedreserved
SQLSTATEnon-reservedreserved
SQLWARNINGnon-reservedreserved
SQRTnon-reservedreserved
STARTnon-reservedreserved
STATEnon-reserved
STATICreserved
STATEnon-reservednon-reserved
STATICnon-reservedreserved
STATISTICSnon-reservednon-reserved
STDDEV_POPreserved
STDDEV_SAMPreserved
STDDEV_POPnon-reservedreserved
STDDEV_SAMPnon-reservedreserved
STOREDnon-reservednon-reserved
STRATIFYnon-reservednon-reserved
STRUCTnon-reservednon-reserved
SUBMULTISETreserved
SUBSETnon-reserved
SUBSTRINGreserved
SUBSTRING_REGEXreserved
SUCCEEDSreserved
SUMreserved
SYMMETRICreserved
SYSTEMreserved
SYSTEM_TIMEreserved
SYSTEM_USERreserved
SUBMULTISETnon-reservedreserved
SUBSETnon-reservednon-reserved
SUBSTRINGnon-reservedreserved
SUBSTRING_REGEXnon-reservedreserved
SUCCEEDSnon-reservedreserved
SUMnon-reservedreserved
SYMMETRICnon-reservedreserved
SYSTEMnon-reservedreserved
SYSTEM_TIMEnon-reservedreserved
SYSTEM_USERnon-reservedreserved
TABLEreservedreserved
TABLESnon-reservednon-reserved
TABLESAMPLEnon-reservedreserved
TANnon-reserved
TANHnon-reserved
TANnon-reservednon-reserved
TANHnon-reservednon-reserved
TBLPROPERTIESnon-reservednon-reserved
TEMPORARYnon-reservednon-reserved
TERMINATEDnon-reservednon-reserved
THENreservedreserved
TIMEreserved
TIMESTAMPreserved
TIMEZONE_HOURreserved
TIMEZONE_MINUTEreserved
TIMEnon-reservedreserved
TIMESTAMPnon-reservedreserved
TIMEZONE_HOURnon-reservedreserved
TIMEZONE_MINUTEnon-reservedreserved
TOreservedreserved
TOUCHnon-reservednon-reserved
TRAILINGreservedreserved
TRANSACTIONnon-reservednon-reserved
TRANSACTIONSnon-reservednon-reserved
TRANSFORMnon-reservednon-reserved
TRANSLATEreserved
TRANSLATE_REGEXreserved
TRANSLATIONreserved
TREATreserved
TRIGGERreserved
TRIMreserved
TRIM_ARRAYreserved
TRANSLATEnon-reservedreserved
TRANSLATE_REGEXnon-reservedreserved
TRANSLATIONnon-reservedreserved
TREATnon-reservedreserved
TRIGGERnon-reservedreserved
TRIMnon-reservedreserved
TRIM_ARRAYnon-reservedreserved
TRUEnon-reservedreserved
TRUNCATEnon-reservedreserved
UESCAPEreserved
UESCAPEnon-reservedreserved
UNARCHIVEnon-reservednon-reserved
UNBOUNDEDnon-reservednon-reserved
UNCACHEnon-reservednon-reserved
UNDERnon-reserved
UNDOreserved
UNDERnon-reservednon-reserved
UNDOnon-reservedreserved
UNIONreservedreserved
UNIQUEreservedreserved
UNKNOWNreserved
UNKNOWNnon-reservedreserved
UNLOCKnon-reservednon-reserved
UNNESTreserved
UNNESTnon-reservedreserved
UNSETnon-reservednon-reserved
UNTILreserved
UPDATEreserved
UPPERreserved
USAGEnon-reserved
UNTILnon-reservedreserved
UPDATEnon-reservedreserved
UPPERnon-reservedreserved
USAGEnon-reservednon-reserved
USEnon-reservednon-reserved
USERreservedreserved
USINGreservedreserved
VALUEreserved
VALUEnon-reservedreserved
VALUESnon-reservedreserved
VALUE_OFreserved
VAR_POPreserved
VAR_SAMPreserved
VARBINARYreserved
VARCHARreserved
VALUE_OFnon-reservedreserved
VAR_POPnon-reservedreserved
VAR_SAMPnon-reservedreserved
VARBINARYnon-reservedreserved
VARCHARnon-reservedreserved
VARYINGreservedreserved
VERSIONINGreserved
VERSIONINGnon-reservedreserved
VIEWnon-reservednon-reserved
WHENreservedreserved
WHENEVERreserved
WHENEVERnon-reservedreserved
WHEREreservedreserved
WHILEreserved
WIDTH_BUCKETreserved
WHILEnon-reservedreserved
WIDTH_BUCKETnon-reservedreserved
WINDOWnon-reservedreserved
WITHreservedreserved
WITHINreserved
WITHOUTreserved
WORKnon-reserved
WRITEnon-reserved
YEARreserved
ZONEnon-reserved
WITHINnon-reservedreserved
WITHOUTnon-reservedreserved
WORKnon-reservednon-reserved
WRITEnon-reservednon-reserved
YEARnon-reservedreserved
ZONEnon-reservednon-reserved
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 49d5c2282cd70..022b869dc215c 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -724,16 +724,17 @@ qualifiedName ; identifier - : reserved + : {ansi}? ansiReserved | strictIdentifier | ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON | UNION | INTERSECT | EXCEPT | SETMINUS ; strictIdentifier - : IDENTIFIER #unquotedIdentifier - | quotedIdentifier #quotedIdentifierAlternative - | nonReserved #unquotedIdentifier + : IDENTIFIER #unquotedIdentifier + | quotedIdentifier #quotedIdentifierAlternative + | {ansi}? ansiNonReserved #unquotedIdentifier + | {!ansi}? nonReserved #unquotedIdentifier ; quotedIdentifier @@ -750,7 +751,7 @@ number | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral ; -reserved +ansiReserved // A list of reserved keywords in SparkSQL. These keywords are reserved in all the ANSI SQL standards // (SQL-92, SQL-99, SQL-2003, SQL-2008, SQL-2011, and SQL-2016) and PostgreSQL. : ALL | AND | ANY | AS | AUTHORIZATION | BOTH | CASE | CAST | CHECK | COLLATE | COLUMN | CONSTRAINT | CREATE @@ -761,25 +762,43 @@ reserved | USING | VARYING | WHEN | WHERE | WITH ; -nonReserved - : ADD | AFTER | ALL | ALTER | ANALYZE | AND | ANY | ARCHIVE | ARRAY | AS | ASC | AT | BETWEEN | BOTH - | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN - | COLLECTION | COLUMN | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST - | CREATE | CUBE | CURRENT | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED - | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY | DISTINCT | DISTRIBUTE | DIV | DROP | ELSE | END - | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT | EXTENDED | EXTERNAL | EXTRACT | FALSE | FIELDS - | FILEFORMAT | FIRST | FOLLOWING | FOR | FORMAT | FORMATTED | FROM | FUNCTION | FUNCTIONS | GLOBAL - | GRANT | GROUP | GROUPING | HAVING | IF | IGNORE | IMPORT | IN | INDEX | INDEXES | INPATH | INPUTFORMAT - | INSERT | INTERVAL | INTO | IS |LIKE | ITEMS | KEYS | LAST | LATERAL | LAZY | LEADING | LIMIT | LINES - | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO | MAP | MSCK | NO | NOT | NULL | NULLS - | OF | OPTION | OPTIONS | OR | ORDER | OUT | OUTER | OUTPUTFORMAT | OVER | OVERWRITE | PARTITION - | PARTITIONED | PARTITIONS | PERCENTLIT | PIVOT | POSITION | PRECEDING | PRINCIPALS | PURGE | RANGE +ansiNonReserved + : ADD | AFTER | ALTER | ANALYZE | ARCHIVE | ARRAY | ASC | AT | BETWEEN | BUCKET | BUCKETS | BY | CACHE + | CASCADE | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLECTION | COLUMNS | COMMENT | COMMIT + | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST | CUBE | CURRENT | DATA | DATABASE | DATABASES + | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY | DISTRIBUTE + | DIV | DROP | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT | EXTENDED | EXTERNAL | EXTRACT | FIELDS + | FILEFORMAT | FIRST | FOLLOWING | FORMAT | FORMATTED | FUNCTION | FUNCTIONS | GLOBAL | GROUPING | IF + | IGNORE | IMPORT | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | ITEMS | KEYS | LAST + | LATERAL | LAZY | LIKE | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO + | MAP | MSCK | NO | NULLS | OF | OPTION | OPTIONS | OUT | OUTPUTFORMAT | OVER | OVERWRITE | PARTITION + | PARTITIONED | PARTITIONS | PERCENT | PERCENTLIT | PIVOT | PRECEDING | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFRESH | RENAME | REPAIR | REPLACE | RESET | RESTRICT - | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SELECT | SEPARATED | SERDE + | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SCHEMA | SEPARATED | SERDE | SERDEPROPERTIES | SET | SETS | SHOW | SKEWED | SORT | SORTED | START | STATISTICS | STORED | STRATIFY - | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | THEN | TO | TOUCH - | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE - | UNLOCK | UNSET | USE | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH + | STRUCT | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | TOUCH | TRANSACTION | TRANSACTIONS + | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE | UNLOCK | UNSET | USE | VALUES | VIEW | WINDOW + ; + +nonReserved + : ADD | AFTER | ALL | ALTER | ANALYZE | AND | ANY | ARCHIVE | ARRAY | AS | ASC | AT | AUTHORIZATION | BETWEEN + | BOTH | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN + | COLLECTION | COLUMN | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST + | CREATE | CUBE | CURRENT | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DATA | DATABASE + | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY + | DISTINCT | DISTRIBUTE | DIV | DROP | ELSE | END | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT | EXTENDED + | EXTERNAL | EXTRACT | FALSE | FETCH | FOREIGN | FIELDS | FILEFORMAT | FIRST | FOLLOWING | FOR | FORMAT + | FORMATTED | FROM | FUNCTION | FUNCTIONS | GLOBAL | GRANT | GROUP | GROUPING | HAVING | IF | IGNORE | IMPORT + | IN | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | INTO | IS |LIKE | ITEMS | KEYS | LAST + | LATERAL | LAZY | LEADING | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO + | MAP | MSCK | NO | NOT | NULL | NULLS | OF | OPTION | OPTIONS | OR | ORDER | OUT | OUTER | OUTPUTFORMAT + | OVER | OVERWRITE | PARTITION | PARTITIONED | PARTITIONS | PERCENTLIT | PIVOT | POSITION | PRECEDING + | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFRESH | RENAME | REPAIR + | REPLACE | RESET | RESTRICT | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SELECT + | SEPARATED | SERDE | SERDEPROPERTIES | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED | START | STATISTICS + | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | THEN + | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED + | UNCACHE | UNLOCK | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; SELECT: 'SELECT'; diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index e4878c87d920e..853068a6cdf58 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1408,11 +1408,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitIdentifier(ctx: IdentifierContext): String = withOrigin(ctx) { val keyword = ctx.getText - if (ctx.reserved() != null) { - if (ansi) { - throw new ParseException( - s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) - } + if (ansi && ctx.ansiReserved() != null) { + throw new ParseException( + s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) } keyword } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index cb8ab61f47ebd..d9526ca463a6c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -59,7 +59,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "natural", "union", "intersect", "except", "database", "on", "join", "cross", "select", "from", "where", "having", "from", "to", "table", "with", "not") - val sparkSQLReservedKeywords = Seq("all", "and", "any", "as", "authorization", "both", "case", + val ansiReservedKeywords = Seq("all", "and", "any", "as", "authorization", "both", "case", "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", @@ -68,6 +68,69 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "varying", "when", "where", "with") + val ansiNonReservedKeywords = Seq("abs", "absolute", "acos", "action", "add", "after", "allocate", + "alter", "analyze", "are", "archive", "array", "array_agg", "array_max_cardinality", "asc", + "asensitive", "asin", "assertion", "asymmetric", "at", "atan", "atomic", "avg", "before", + "begin", "begin_frame", "begin_partition", "between", "bigint", "binary", "bit", "bit_length", + "blob", "boolean", "breadth", "bucket", "buckets", "by", "cache", "call", "called", + "cardinality", "cascade", "cascaded", "catalog", "ceil", "ceiling", "change", "char", + "char_length", "character", "character_length", "classifier", "clear", "clob", "close", + "cluster", "clustered", "coalesce", "codegen", "collation", "collect", "collection", + "columns", "comment", "commit", "compact", "compactions", "compute", "concatenate", "condition", + "connect", "connection", "constraints", "constructor", "contains", "continue", "convert", + "copy", "corr", "corresponding", "cos", "cosh", "cost", "count", "covar_pop", "covar_samp", + "cube", "cume_dist", "current", "current_catalog", "current_default_transform_group", + "current_path", "current_role", "current_row", "current_schema", + "current_transform_group_for_type", "cursor", "cycle", "data", "database", "databases", "date", + "day", "dbproperties", "deallocate", "dec", "decfloat", "decimal", "declare", "default", + "deferrable", "deferred", "define", "defined", "delete", "delimited", "dense_rank", "depth", + "deref", "desc", "describe", "descriptor", "deterministic", "dfs", "diagnostics", + "directories", "directory", "disconnect", "distribute", "div", "do", "domain", "double", + "drop", "dynamic", "each", "element", "elseif", "empty", "end_frame", "end_partition", + "equals", "escape", "escaped", "every", "exception", "exchange", "exec", "execute", + "exists", "exit", "exp", "explain", "export", "extended", "external", "extract", "fields", + "fileformat", "filter", "first", "first_value", "float", "following", "format", "formatted", + "found", "frame_row", "free", "function", "functions", "fusion", "general", "get", "global", + "go", "goto", "grouping", "groups", "handler", "hold", "hour", "identity", "if", "ignore", + "immediate", "import", "indicator", "index", "indexes", "initial", "initially", "inout", + "inpath", "input", "inputformat", "insensitive", "insert", "int", "integer", "intersection", + "interval", "isolation", "items", "iterate", "json_array", "json_arrayagg", "json_exists", + "json_object", "json_objectagg", "json_query", "json_table", "json_table_primitive", + "json_value", "key", "keys", "lag", "language", "large", "last", "last_value", "lateral", + "lazy", "lead", "leave", "level", "like", "like_regex", "limit", "lines", "list", "listagg", + "ln", "load", "local", "localtime", "localtimestamp", "location", "locator", "lock", "locks", + "log", "log10", "logical", "loop", "lower", "macro", "map", "match", "match_number", + "match_recognize", "matches", "max", "member", "merge", "method", "min", "minute", "mod", + "modifies", "module", "month", "msck", "multiset", "names", "national", "nchar", "nclob", + "new", "next", "no", "none", "normalize", "nth_value", "ntile", "nulls", "nullif", "numeric", + "object", "occurrences_regex", "octet_length", "of", "offset", "old", "omit", "one", "open", + "option", "options", "ordinality", "out", "output", "outputformat", "over", "overlay", + "overwrite", "pad", "parameter", "partial", "partition", "partitioned", "partitions", "path", + "pattern", "per", "percent", "percent_rank", "percentile_cont", "percentile_disc", "percentlit", + "period", "pivot", "portion", "power", "precedes", "preceding", "precision", "prepare", + "preserve", "principals", "prior", "privileges", "procedure", "ptf", "public", "purge", "range", + "rank", "read", "reads", "real", "recordreader", "recordwriter", "recursive", "recover", + "reduce", "ref", "referencing", "refresh", "regr_avgx", "regr_avgy", "regr_count", + "regr_intercept", "regr_r2", "regr_slope", "regr_sxx", "regr_sxy", "regr_syy", "relative", + "release", "rename", "repair", "repeat", "replace", "reset", "resignal", "restrict", "result", + "return", "returns", "revoke", "rlike", "role", "roles", "rollback", "rollup", "routine", + "row", "row_number", "rows", "running", "savepoint", "schema", "scope", "scroll", "search", + "second", "section", "seek", "sensitive", "separated", "serde", "serdeproperties", "session", + "set", "sets", "show", "signal", "similar", "sin", "sinh", "size", "skip", "skewed", "smallint", + "sort", "sorted", "space", "specific", "specifictype", "sql", "sqlcode", "sqlerror", + "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", "state", "static", "statistics", + "stddev_pop", "stddev_samp", "stored", "stratify", "struct", "submultiset", "subset", + "substring", "substring_regex", "succeeds", "sum", "symmetric", "system", "system_time", + "system_user", "tables", "tablesample", "tan", "tanh", "tblproperties", "temporary", + "terminated", "time", "timestamp", "timezone_hour", "timezone_minute", "touch", "transaction", + "transactions", "transform", "translate", "translate_regex", "translation", "treat", "trigger", + "trim", "trim_array", "true", "truncate", "uescape", "unarchive", "unbounded", "uncache", + "under", "undo", "unknown", "unlock", "unnest", "unset", "until", "update", "upper", "usage", + "use", "value", "values", "value_of", "var_pop", "var_samp", "varbinary", "varchar", + "versioning", "view", "whenever", "while", "width_bucket", "window", "within", "without", + "work", "write", "year", "zone" + ) + test("table identifier") { // Regular names. assert(TableIdentifier("q") === parseTableIdentifier("q")) @@ -87,9 +150,9 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { assert(TableIdentifier("x.y.z", None) === parseTableIdentifier("`x.y.z`")) } - test("table identifier - reserved keywords") { + test("table identifier - reserved/non-reserved keywords if ANSI mode enabled") { withSQLConf(SQLConf.ANSI_SQL_PARSER.key -> "true") { - sparkSQLReservedKeywords.foreach { keyword => + ansiReservedKeywords.foreach { keyword => val errMsg = intercept[ParseException] { parseTableIdentifier(keyword) }.getMessage @@ -98,11 +161,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) } - } - - // If an ANSI mode disabled, we can use these keywords as identifiers - withSQLConf(SQLConf.ANSI_SQL_PARSER.key -> "false") { - sparkSQLReservedKeywords.foreach { keyword => + ansiNonReservedKeywords.foreach { keyword => assert(TableIdentifier(keyword) === parseTableIdentifier(s"$keyword")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.$keyword")) } From ffd07711466fbbfd7d3fab9d0427cd78a857340d Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Sun, 17 Feb 2019 10:56:37 +0900 Subject: [PATCH 04/18] Move VARYING to non-reserved --- docs/sql-reserved-and-non-reserved-key-words.md | 2 +- .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 3 +-- .../sql/catalyst/parser/TableIdentifierParserSuite.scala | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index d54961b98e079..03aecbd56fbc0 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -547,7 +547,7 @@ A full list of reserved/non-reserved key words is as follows: VAR_SAMPnon-reservedreserved VARBINARYnon-reservedreserved VARCHARnon-reservedreserved - VARYINGreservedreserved + VARYINGnon-reservedreserved VERSIONINGnon-reservedreserved VIEWnon-reservednon-reserved WHENreservedreserved diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 022b869dc215c..56b0ff9b2e75b 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -759,7 +759,7 @@ ansiReserved | FALSE | FETCH | FOR | FOREIGN | FROM | FULL | GRANT | GROUP | HAVING | IN | INNER | INTERSECT | INTO | IS | JOIN | LEADING | LEFT | NATURAL | NOT | NULL | ON | ONLY | OR | ORDER | OUTER | OVERLAPS | PRIMARY | REFERENCES | RIGHT | SELECT | SESSION_USER | SOME | TABLE | THEN | TO | TRAILING | UNION | UNIQUE | USER - | USING | VARYING | WHEN | WHERE | WITH + | USING | WHEN | WHERE | WITH ; ansiNonReserved @@ -1053,7 +1053,6 @@ SESSION_USER: 'SESSION_USER'; SOME: 'SOME'; UNIQUE: 'UNIQUE'; USER: 'USER'; -VARYING: 'VARYING'; STRING : '\'' ( ~('\''|'\\') | ('\\' .) )* '\'' diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index d9526ca463a6c..3b30d9ca610e8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -65,8 +65,8 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", "intersect", "into", "is", "join", "leading", "left", "natural", "not", "null", "on", "only", "or", "order", "outer", "overlaps", "primary", "references", "right", "select", "session_user", - "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "varying", - "when", "where", "with") + "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "when", + "where", "with") val ansiNonReservedKeywords = Seq("abs", "absolute", "acos", "action", "add", "after", "allocate", "alter", "analyze", "are", "archive", "array", "array_agg", "array_max_cardinality", "asc", @@ -126,7 +126,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "transactions", "transform", "translate", "translate_regex", "translation", "treat", "trigger", "trim", "trim_array", "true", "truncate", "uescape", "unarchive", "unbounded", "uncache", "under", "undo", "unknown", "unlock", "unnest", "unset", "until", "update", "upper", "usage", - "use", "value", "values", "value_of", "var_pop", "var_samp", "varbinary", "varchar", + "use", "value", "values", "value_of", "var_pop", "var_samp", "varbinary", "varchar", "varying", "versioning", "view", "whenever", "while", "width_bucket", "window", "within", "without", "work", "write", "year", "zone" ) From 04a7a351b7b04a1142f887d86b69fbffa1ef5ad5 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Sun, 17 Feb 2019 11:18:28 +0900 Subject: [PATCH 05/18] Add some keywords in nonReserved --- .../org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 10 +++++----- .../catalyst/parser/TableIdentifierParserSuite.scala | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 56b0ff9b2e75b..19b4432e08dea 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -782,7 +782,7 @@ ansiNonReserved nonReserved : ADD | AFTER | ALL | ALTER | ANALYZE | AND | ANY | ARCHIVE | ARRAY | AS | ASC | AT | AUTHORIZATION | BETWEEN - | BOTH | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN + | BOTH | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHECK | COLLATE | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLECTION | COLUMN | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST | CREATE | CUBE | CURRENT | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY @@ -791,14 +791,14 @@ nonReserved | FORMATTED | FROM | FUNCTION | FUNCTIONS | GLOBAL | GRANT | GROUP | GROUPING | HAVING | IF | IGNORE | IMPORT | IN | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | INTO | IS |LIKE | ITEMS | KEYS | LAST | LATERAL | LAZY | LEADING | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO - | MAP | MSCK | NO | NOT | NULL | NULLS | OF | OPTION | OPTIONS | OR | ORDER | OUT | OUTER | OUTPUTFORMAT - | OVER | OVERWRITE | PARTITION | PARTITIONED | PARTITIONS | PERCENTLIT | PIVOT | POSITION | PRECEDING - | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFRESH | RENAME | REPAIR + | MAP | MSCK | NO | NOT | NULL | NULLS | OF | ONLY | OPTION | OPTIONS | OR | ORDER | OUT | OUTER | OUTPUTFORMAT + | OVER | OVERLAPS | OVERWRITE | PARTITION | PARTITIONED | PARTITIONS | PERCENTLIT | PIVOT | POSITION | PRECEDING + | PRIMARY | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFERENCES | REFRESH | RENAME | REPAIR | REPLACE | RESET | RESTRICT | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SELECT | SEPARATED | SERDE | SERDEPROPERTIES | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED | START | STATISTICS | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | THEN | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED - | UNCACHE | UNLOCK | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH + | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; SELECT: 'SELECT'; diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index 3b30d9ca610e8..be6af24999e8e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -59,6 +59,9 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "natural", "union", "intersect", "except", "database", "on", "join", "cross", "select", "from", "where", "having", "from", "to", "table", "with", "not") + val otherNonReservedKeywords = Seq( + "check", "collate", "overlaps", "only", "primary", "references", "unique") + val ansiReservedKeywords = Seq("all", "and", "any", "as", "authorization", "both", "case", "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", @@ -179,7 +182,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { test("table identifier - non reserved keywords") { // Hive keywords are allowed. - hiveNonReservedKeywords.foreach { nonReserved => + (hiveNonReservedKeywords ++ otherNonReservedKeywords).foreach { nonReserved => assert(TableIdentifier(nonReserved) === parseTableIdentifier(nonReserved)) } } From 9e91a2f13ff3020c7c1eb02f83bb615fd2c21fe9 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Sun, 17 Feb 2019 13:48:25 +0900 Subject: [PATCH 06/18] Add a new column in the doc --- ...sql-reserved-and-non-reserved-key-words.md | 1110 ++++++++--------- .../spark/sql/catalyst/parser/SqlBase.g4 | 26 +- .../parser/TableIdentifierParserSuite.scala | 91 +- 3 files changed, 648 insertions(+), 579 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index 03aecbd56fbc0..829315b9ab293 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -5,562 +5,562 @@ displayTitle: SQL Reserved/Non-Reserved Key Words --- When `spark.sql.parser.ansi.enabled` is set to true (false by default), some keywords are reserved for Spark SQL. -Reserved key words in Spark SQL basically follow the ANSI SQL-2011 standards, but it is slightly different between each other. +These reserved key words in this ANSI mode basically follow the ANSI SQL-2011 standard, but it is slightly different between each other. A full list of reserved/non-reserved key words is as follows: - - - - - - - - - - - - - - - - - - - - - - non-reserved - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Key WordSpark SQLSQL-2011
ABSnon-reservedreserved
ABSOLUTEnon-reservednon-reserved
ACOSnon-reservednon-reserved
ACTIONnon-reservednon-reserved
ADDnon-reservednon-reserved
AFTERnon-reservednon-reserved
ALLreservedreserved
ALLOCATEnon-reservedreserved
ALTERnon-reservedreserved
ANALYZEnon-reservednon-reserved
ANDreservedreserved
ANYreservedreserved
AREnon-reservedreserved
ARCHIVEnon-reservednon-reserved
ARRAYnon-reservedreserved
ARRAY_AGGnon-reservedreserved
ARRAY_MAX_CARDINALITYnon-reservedreserved
ASreservedreserved
ASCnon-reservednon-reserved
ASENSITIVEnon-reservedreserved
ASINnon-reservedreserved
ASSERTIONnon-reservednon-reserved
ASYMMETRICnon-reservedreserved
ATnon-reservedreserved
ATANnon-reservednon-reserved
ATOMICnon-reservedreserved
AUTHORIZATIONreservedreserved
AVGnon-reservedreserved
BEFOREnon-reservednon-reserved
BEGINnon-reservedreserved
BEGIN_FRAMEnon-reservedreserved
BEGIN_PARTITIONnon-reservedreserved
BETWEENnon-reservedreserved
BIGINTnon-reservedreserved
BINARYnon-reservedreserved
BITnon-reservednon-reserved
BIT_LENGTHnon-reservednon-reserved
BLOBnon-reservedreserved
BOOLEANnon-reservedreserved
BOTHreservedreserved
BREADTHnon-reservednon-reserved
BUCKETnon-reservednon-reserved
BUCKETSnon-reservednon-reserved
BYnon-reservedreserved
CACHEnon-reservednon-reserved
CALLnon-reservedreserved
CALLEDnon-reservedreserved
CARDINALITYnon-reservedreserved
CASCADEnon-reservedreserved
CASCADEDnon-reservedreserved
CASEreservedreserved
CASTreservedreserved
CATALOGnon-reservednon-reserved
CEILnon-reservedreserved
CEILINGnon-reservedreserved
CHANGEnon-reservednon-reserved
CHARnon-reservedreserved
CHAR_LENGTHnon-reservedreserved
CHARACTERnon-reservedreserved
CHARACTER_LENGTHnon-reservedreserved
CHECKreservedreserved
CLASSIFIERnon-reservednon-reserved
CLEARnon-reservednon-reserved
CLOBnon-reservedreserved
CLOSEnon-reservedreserved
CLUSTERnon-reservednon-reserved
CLUSTEREDnon-reservednon-reserved
COALESCEnon-reservedreserved
CODEGENnon-reservednon-reserved
COLLATEreservedreserved
COLLATIONnon-reservednon-reserved
COLLECTnon-reservedreserved
COLLECTIONnon-reservednon-reserved
COLUMNreservedreserved
COLUMNSnon-reservednon-reserved
COMMENTnon-reservednon-reserved
COMMITnon-reservedreserved
COMPACTnon-reservednon-reserved
COMPACTIONSnon-reservednon-reserved
COMPUTEnon-reservednon-reserved
CONCATENATEnon-reservednon-reserved
CONDITIONnon-reservedreserved
CONNECTnon-reservednon-reserved
CONNECTIONnon-reservednon-reserved
CONSTRAINTreservedreserved
CONSTRAINTSnon-reservednon-reserved
CONSTRUCTORnon-reservednon-reserved
CONTAINSnon-reservednon-reserved
CONTINUEnon-reservednon-reserved
CONVERTnon-reservedreserved
COPYnon-reservednon-reserved
CORRnon-reservedreserved
CORRESPONDINGnon-reservedreserved
COSnon-reservednon-reserved
COSHnon-reservednon-reserved
COSTnon-reservednon-reserved
COUNTnon-reservedreserved
COVAR_POPnon-reservedreserved
COVAR_SAMPnon-reservedreserved
CREATEreservedreserved
CROSSreservedreserved
CUBEnon-reservedreserved
CUME_DISTnon-reservedreserved
CURRENTnon-reservedreserved
CURRENT_CATALOGnon-reservedreserved
CURRENT_DATEreservedreserved
CURRENT_DEFAULT_TRANSFORM_GROUPnon-reservedreserved
CURRENT_PATHnon-reservedreserved
CURRENT_ROLEnon-reservedreserved
CURRENT_ROWnon-reservedreserved
CURRENT_SCHEMAnon-reservedreserved
CURRENT_TIMEreservedreserved
CURRENT_TIMESTAMPreservedreserved
CURRENT_TRANSFORM_GROUP_FOR_TYPEnon-reservedreserved
CURRENT_USERreservedreserved
CURSORnon-reservedreserved
CYCLEnon-reservedreserved
DATAnon-reservednon-reserved
DATABASEnon-reservednon-reserved
DATABASESnon-reservednon-reserved
DATEnon-reservedreserved
DAYnon-reservedreserved
DBPROPERTIESnon-reservednon-reserved
DEALLOCATEnon-reservedreserved
DECnon-reservedreserved
DECFLOATnon-reservednon-reserved
DECIMALnon-reservedreserved
DECLAREnon-reservedreserved
DEFAULTnon-reservedreserved
DEFERRABLEnon-reservednon-reserved
DEFERREDnon-reservednon-reserved
DEFINEnon-reservednon-reserved
DEFINEDnon-reservednon-reserved
DELETEnon-reservedreserved
DELIMITEDnon-reservednon-reserved
DENSE_RANKnon-reservedreserved
DEPTHnon-reservednon-reserved
DEREFnon-reservedreserved
DESCnon-reservednon-reserved
DESCRIBEnon-reservedreserved
DESCRIPTORnon-reservednon-reserved
DETERMINISTICnon-reservedreserved
DFSnon-reservednon-reserved
DIAGNOSTICSnon-reservednon-reserved
DIRECTORIESnon-reservednon-reserved
DIRECTORYnon-reservednon-reserved
DISCONNECTnon-reservedreserved
DISTINCTreservedreserved
DISTRIBUTEnon-reservednon-reserved
DIVnon-reservednon-reserved
DOnon-reservedreserved
DOMAINnon-reservednon-reserved
DOUBLEnon-reservedreserved
DROPnon-reservedreserved
DYNAMICnon-reservedreserved
EACHnon-reservedreserved
ELEMENTnon-reservedreserved
ELSEreservedreserved
ELSEIFnon-reservedreserved
EMPTYnon-reservednon-reserved
ENDreservedreserved
END_FRAMEnon-reservedreserved
END_PARTITIONnon-reservedreserved
EQUALSnon-reservednon-reserved
ESCAPEnon-reservedreserved
ESCAPEDnon-reservednon-reserved
EVERYnon-reservedreserved
EXCEPTreservedreserved
EXCEPTIONnon-reservednon-reserved
EXCHANGEnon-reservednon-reserved
EXECnon-reservedreserved
EXECUTEnon-reservedreserved
EXISTSnon-reservedreserved
EXITnon-reservednon-reserved
EXPnon-reservednon-reserved
EXPLAINnon-reservednon-reserved
EXPORTnon-reservednon-reserved
EXTENDEDnon-reservednon-reserved
EXTERNALnon-reservedreserved
EXTRACTnon-reservedreserved
FALSEreservedreserved
FETCHreservedreserved
FIELDSnon-reservednon-reserved
FILEFORMATnon-reservednon-reserved
FILTERnon-reservedreserved
FIRSTnon-reservednon-reserved
FIRST_VALUEnon-reservedreserved
FLOATnon-reservedreserved
FOLLOWINGnon-reservednon-reserved
FORreservedreserved
FOREIGNreservedreserved
FORMATnon-reservednon-reserved
FORMATTEDnon-reservednon-reserved
FOUNDnon-reservednon-reserved
FRAME_ROWnon-reservedreserved
FREEnon-reservedreserved
FROMreservedreserved
FULLreservedreserved
FUNCTIONnon-reservedreserved
FUNCTIONSnon-reservednon-reserved
FUSIONnon-reservednon-reserved
GENERALnon-reservednon-reserved
GETnon-reservedreserved
GLOBALnon-reservedreserved
GOnon-reservednon-reserved
GOTOnon-reservednon-reserved
GRANTreservedreserved
GROUPreservedreserved
GROUPINGnon-reservedreserved
GROUPSnon-reservedreserved
HANDLERnon-reservedreserved
HAVINGreservedreserved
HOLDnon-reservedreserved
HOURnon-reservedreserved
IDENTITYnon-reservedreserved
IFnon-reservedreserved
IGNOREnon-reservednon-reserved
IMMEDIATEnon-reservednon-reserved
IMPORTnon-reservednon-reserved
INreservedreserved
INDICATORnon-reservedreserved
INDEXnon-reservednon-reserved
INDEXESnon-reservednon-reserved
INITIALnon-reservednon-reserved
INITIALLYnon-reservednon-reserved
INNERreservedreserved
INOUTnon-reservedreserved
INPATHnon-reservednon-reserved
INPUTnon-reservednon-reserved
INPUTFORMATnon-reservednon-reserved
INSENSITIVEnon-reservedreserved
INSERTnon-reservedreserved
INTnon-reservedreserved
INTEGERnon-reservedreserved
INTERSECTreservedreserved
INTERSECTIONnon-reservedreserved
INTERVALnon-reservedreserved
INTOreservedreserved
ISreservedreserved
ISOLATIONnon-reservednon-reserved
ITEMSnon-reservednon-reserved
ITERATEnon-reservedreserved
JOINreservedreserved
JSON_ARRAYnon-reservednon-reserved
JSON_ARRAYAGGnon-reservednon-reserved
JSON_EXISTSnon-reservednon-reserved
JSON_OBJECTnon-reservednon-reserved
JSON_OBJECTAGGnon-reservednon-reserved
JSON_QUERYnon-reservednon-reserved
JSON_TABLEnon-reservednon-reserved
JSON_TABLE_PRIMITIVEnon-reservednon-reserved
JSON_VALUEnon-reservednon-reserved
KEYnon-reservednon-reserved
KEYSnon-reservednon-reserved
LAGnon-reservednon-reserved
LANGUAGEnon-reservedreserved
LARGEnon-reservedreserved
LASTnon-reservednon-reserved
LAST_VALUEnon-reservedreserved
LATERALnon-reservedreserved
LAZYnon-reservednon-reserved
LEADnon-reservedreserved
LEADINGreservedreserved
LEAVEnon-reservedreserved
LEFTreservedreserved
LEVELnon-reservednon-reserved
LIKEnon-reservedreserved
LIKE_REGEXnon-reservedreserved
LIMITnon-reservednon-reserved
LINESnon-reservednon-reserved
LISTnon-reservednon-reserved
LISTAGGnon-reservednon-reserved
LNnon-reservedreserved
LOADnon-reservednon-reserved
LOCALnon-reservedreserved
LOCALTIMEnon-reservedreserved
LOCALTIMESTAMPnon-reservedreserved
LOCATIONnon-reservednon-reserved
LOCATORnon-reservednon-reserved
LOCKnon-reservednon-reserved
LOCKSnon-reservednon-reserved
LOGnon-reservednon-reserved
LOG10non-reservednon-reserved
LOGICALnon-reservednon-reserved
LOOPnon-reservedreserved
LOWERnon-reservedreserved
MACROnon-reservednon-reserved
MAPnon-reservednon-reserved
MATCHnon-reservedreserved
MATCH_NUMBERnon-reservednon-reserved
MATCH_RECOGNIZEnon-reservednon-reserved
MATCHESnon-reservednon-reserved
MAXnon-reservedreserved
MEMBERnon-reservedreserved
MERGEnon-reservedreserved
METHODnon-reservedreserved
MINnon-reservedreserved
MINUTEnon-reservedreserved
MODnon-reservedreserved
MODIFIESnon-reservedreserved
MODULEnon-reservedreserved
MONTHnon-reservedreserved
MSCKnon-reservednon-reserved
MULTISETnon-reservedreserved
NAMESnon-reservednon-reserved
NATIONALnon-reservedreserved
NATURALreservedreserved
NCHARnon-reservedreserved
NCLOBnon-reservedreserved
NEWnon-reservedreserved
NEXTnon-reservednon-reserved
NOnon-reservedreserved
NONEnon-reservedreserved
NORMALIZEnon-reservedreserved
NOTreservedreserved
NTH_VALUEnon-reservedreserved
NTILEnon-reservedreserved
NULLreservedreserved
NULLSnon-reservednon-reserved
NULLIFnon-reservedreserved
NUMERICnon-reservedreserved
OBJECTnon-reservednon-reserved
OCCURRENCES_REGEXnon-reservednon-reserved
OCTET_LENGTHnon-reservedreserved
OFnon-reservedreserved
OFFSETnon-reservedreserved
OLDnon-reservedreserved
OMITnon-reservednon-reserved
ONreservedreserved
ONEnon-reservednon-reserved
ONLYreservedreserved
OPENnon-reservedreserved
OPTIONnon-reservednon-reserved
OPTIONSnon-reservednon-reserved
ORreservedreserved
ORDERreservedreserved
ORDINALITYnon-reservednon-reserved
OUTnon-reservedreserved
OUTERreservedreserved
OUTPUTnon-reservednon-reserved
OUTPUTFORMATnon-reservednon-reserved
OVERnon-reservednon-reserved
OVERLAPSreservedreserved
OVERLAYnon-reservedreserved
OVERWRITEnon-reservednon-reserved
PADnon-reservednon-reserved
PARAMETERnon-reservedreserved
PARTIALnon-reservednon-reserved
PARTITIONnon-reservedreserved
PARTITIONEDnon-reservednon-reserved
PARTITIONSnon-reservednon-reserved
PATHnon-reservednon-reserved
PATTERNnon-reservednon-reserved
PERnon-reservednon-reserved
PERCENTnon-reservedreserved
PERCENT_RANKnon-reservedreserved
PERCENTILE_CONTnon-reservedreserved
PERCENTILE_DISCnon-reservedreserved
PERCENTLITnon-reservednon-reserved
PERIODnon-reservedreserved
PIVOTnon-reservednon-reserved
PORTIONnon-reservedreserved
POSITIONnon-reservedreserved
POSITION_REGEXnon-reservedreserved
POWERnon-reservedreserved
PRECEDESnon-reservedreserved
PRECEDINGnon-reservednon-reserved
PRECISIONnon-reservedreserved
PREPAREnon-reservedreserved
PRESERVEnon-reservednon-reserved
PRIMARYreservedreserved
PRINCIPALSnon-reservednon-reserved
PRIORnon-reservednon-reserved
PRIVILEGESnon-reservednon-reserved
PROCEDUREnon-reservedreserved
PTFnon-reservednon-reserved
PUBLICnon-reservednon-reserved
PURGEnon-reservednon-reserved
RANGEnon-reservedreserved
RANKnon-reservedreserved
READnon-reservednon-reserved
READSnon-reservedreserved
REALnon-reservedreserved
RECORDREADERnon-reservednon-reserved
RECORDWRITERnon-reservednon-reserved
RECURSIVEnon-reservedreserved
RECOVERnon-reservednon-reserved
REDUCEnon-reservednon-reserved
REFnon-reservedreserved
REFERENCESreservedreserved
REFERENCINGnon-reservedreserved
REFRESHnon-reservednon-reserved
REGR_AVGXnon-reservedreserved
REGR_AVGYnon-reservedreserved
REGR_COUNTnon-reservedreserved
REGR_INTERCEPTnon-reservedreserved
REGR_R2non-reservedreserved
REGR_SLOPEnon-reservedreserved
REGR_SXXnon-reservedreserved
REGR_SXYnon-reservedreserved
REGR_SYYnon-reservedreserved
RELATIVEnon-reservednon-reserved
RELEASEnon-reservedreserved
RENAMEnon-reservednon-reserved
REPAIRnon-reservednon-reserved
REPEATnon-reservedreserved
REPLACEnon-reservednon-reserved
RESETnon-reservednon-reserved
RESIGNALnon-reservedreserved
RESTRICTnon-reservednon-reserved
RESULTnon-reservedreserved
RETURNnon-reservedreserved
RETURNSnon-reservedreserved
REVOKEnon-reservedreserved
RIGHTreservedreserved
RLIKEnon-reservednon-reserved
ROLEnon-reservednon-reserved
ROLESnon-reservednon-reserved
ROLLBACKnon-reservedreserved
ROLLUPnon-reservedreserved
ROUTINEnon-reservednon-reserved
ROWnon-reservedreserved
ROW_NUMBERnon-reservedreserved
ROWSnon-reservedreserved
RUNNINGnon-reservednon-reserved
SAVEPOINTnon-reservedreserved
SCHEMAnon-reservednon-reserved
SCOPEnon-reservedreserved
SCROLLnon-reservedreserved
SEARCHnon-reservedreserved
SECONDnon-reservedreserved
SECTIONnon-reservednon-reserved
SEEKnon-reservednon-reserved
SELECTreservedreserved
SENSITIVEnon-reservedreserved
SEPARATEDnon-reservednon-reserved
SERDEnon-reservednon-reserved
SERDEPROPERTIESnon-reservednon-reserved
SESSIONnon-reservednon-reserved
SESSION_USERreservedreserved
SETnon-reservedreserved
SETSnon-reservednon-reserved
SHOWnon-reservednon-reserved
SIGNALnon-reservedreserved
SIMILARnon-reservedreserved
SINnon-reservednon-reserved
SINHnon-reservednon-reserved
SIZEnon-reservednon-reserved
SKIPnon-reservednon-reserved
SKEWEDnon-reservednon-reserved
SMALLINTnon-reservedreserved
SOMEreservedreserved
SORTnon-reservednon-reserved
SORTEDnon-reservednon-reserved
SPACEnon-reservednon-reserved
SPECIFICnon-reservedreserved
SPECIFICTYPEnon-reservedreserved
SQLnon-reservedreserved
SQLCODEnon-reservednon-reserved
SQLERRORnon-reservednon-reserved
SQLEXCEPTIONnon-reservedreserved
SQLSTATEnon-reservedreserved
SQLWARNINGnon-reservedreserved
SQRTnon-reservedreserved
STARTnon-reservedreserved
STATEnon-reservednon-reserved
STATICnon-reservedreserved
STATISTICSnon-reservednon-reserved
STDDEV_POPnon-reservedreserved
STDDEV_SAMPnon-reservedreserved
STOREDnon-reservednon-reserved
STRATIFYnon-reservednon-reserved
STRUCTnon-reservednon-reserved
SUBMULTISETnon-reservedreserved
SUBSETnon-reservednon-reserved
SUBSTRINGnon-reservedreserved
SUBSTRING_REGEXnon-reservedreserved
SUCCEEDSnon-reservedreserved
SUMnon-reservedreserved
SYMMETRICnon-reservedreserved
SYSTEMnon-reservedreserved
SYSTEM_TIMEnon-reservedreserved
SYSTEM_USERnon-reservedreserved
TABLEreservedreserved
TABLESnon-reservednon-reserved
TABLESAMPLEnon-reservedreserved
TANnon-reservednon-reserved
TANHnon-reservednon-reserved
TBLPROPERTIESnon-reservednon-reserved
TEMPORARYnon-reservednon-reserved
TERMINATEDnon-reservednon-reserved
THENreservedreserved
TIMEnon-reservedreserved
TIMESTAMPnon-reservedreserved
TIMEZONE_HOURnon-reservedreserved
TIMEZONE_MINUTEnon-reservedreserved
TOreservedreserved
TOUCHnon-reservednon-reserved
TRAILINGreservedreserved
TRANSACTIONnon-reservednon-reserved
TRANSACTIONSnon-reservednon-reserved
TRANSFORMnon-reservednon-reserved
TRANSLATEnon-reservedreserved
TRANSLATE_REGEXnon-reservedreserved
TRANSLATIONnon-reservedreserved
TREATnon-reservedreserved
TRIGGERnon-reservedreserved
TRIMnon-reservedreserved
TRIM_ARRAYnon-reservedreserved
TRUEnon-reservedreserved
TRUNCATEnon-reservedreserved
UESCAPEnon-reservedreserved
UNARCHIVEnon-reservednon-reserved
UNBOUNDEDnon-reservednon-reserved
UNCACHEnon-reservednon-reserved
UNDERnon-reservednon-reserved
UNDOnon-reservedreserved
UNIONreservedreserved
UNIQUEreservedreserved
UNKNOWNnon-reservedreserved
UNLOCKnon-reservednon-reserved
UNNESTnon-reservedreserved
UNSETnon-reservednon-reserved
UNTILnon-reservedreserved
UPDATEnon-reservedreserved
UPPERnon-reservedreserved
USAGEnon-reservednon-reserved
USEnon-reservednon-reserved
USERreservedreserved
USINGreservedreserved
VALUEnon-reservedreserved
VALUESnon-reservedreserved
VALUE_OFnon-reservedreserved
VAR_POPnon-reservedreserved
VAR_SAMPnon-reservedreserved
VARBINARYnon-reservedreserved
VARCHARnon-reservedreserved
VARYINGnon-reservedreserved
VERSIONINGnon-reservedreserved
VIEWnon-reservednon-reserved
WHENreservedreserved
WHENEVERnon-reservedreserved
WHEREreservedreserved
WHILEnon-reservedreserved
WIDTH_BUCKETnon-reservedreserved
WINDOWnon-reservedreserved
WITHreservedreserved
WITHINnon-reservedreserved
WITHOUTnon-reservedreserved
WORKnon-reservednon-reserved
WRITEnon-reservednon-reserved
YEARnon-reservedreserved
ZONEnon-reservednon-reserved
Key WordSpark SQL (ANSI mode)Spark SQL (default mode)SQL-2011
ABSnon-reservednon-reservedreserved
ABSOLUTEnon-reservednon-reservednon-reserved
ACOSnon-reservednon-reservednon-reserved
ACTIONnon-reservednon-reservednon-reserved
ADDnon-reservednon-reservednon-reserved
AFTERnon-reservednon-reservednon-reserved
ALLreservednon-reservedreserved
ALLOCATEnon-reservednon-reservedreserved
ALTERnon-reservednon-reservedreserved
ANALYZEnon-reservednon-reservednon-reserved
ANDreservednon-reservedreserved
ANYreservednon-reservedreserved
AREnon-reservednon-reservedreserved
ARCHIVEnon-reservednon-reservednon-reserved
ARRAYnon-reservednon-reservedreserved
ARRAY_AGGnon-reservednon-reservedreserved
ARRAY_MAX_CARDINALITYnon-reservednon-reservedreserved
ASreservednon-reservedreserved
ASCnon-reservednon-reservednon-reserved
ASENSITIVEnon-reservednon-reservedreserved
ASINnon-reservednon-reservedreserved
ASSERTIONnon-reservednon-reservednon-reserved
ASYMMETRICnon-reservednon-reservedreserved
ATnon-reservednon-reservedreserved
ATANnon-reservednon-reservednon-reserved
ATOMICnon-reservednon-reservedreserved
AUTHORIZATIONreservednon-reservedreserved
AVGnon-reservednon-reservedreserved
BEFOREnon-reservednon-reservednon-reserved
BEGINnon-reservednon-reservedreserved
BEGIN_FRAMEnon-reservednon-reservedreserved
BEGIN_PARTITIONnon-reservednon-reservedreserved
BETWEENnon-reservednon-reservedreserved
BIGINTnon-reservednon-reservedreserved
BINARYnon-reservednon-reservedreserved
BITnon-reservednon-reservednon-reserved
BIT_LENGTHnon-reservednon-reservednon-reserved
BLOBnon-reservednon-reservedreserved
BOOLEANnon-reservednon-reservedreserved
BOTHreservednon-reservedreserved
BREADTHnon-reservednon-reservednon-reserved
BUCKETnon-reservednon-reservednon-reserved
BUCKETSnon-reservednon-reservednon-reserved
BYnon-reservednon-reservedreserved
CACHEnon-reservednon-reservednon-reserved
CALLnon-reservednon-reservedreserved
CALLEDnon-reservednon-reservedreserved
CARDINALITYnon-reservednon-reservedreserved
CASCADEnon-reservednon-reservedreserved
CASCADEDnon-reservednon-reservedreserved
CASEreservednon-reservedreserved
CASTreservednon-reservedreserved
CATALOGnon-reservednon-reservednon-reserved
CEILnon-reservednon-reservedreserved
CEILINGnon-reservednon-reservedreserved
CHANGEnon-reservednon-reservednon-reserved
CHARnon-reservednon-reservedreserved
CHAR_LENGTHnon-reservednon-reservedreserved
CHARACTERnon-reservednon-reservedreserved
CHARACTER_LENGTHnon-reservednon-reservedreserved
CHECKreservednon-reservedreserved
CLASSIFIERnon-reservednon-reservednon-reserved
CLEARnon-reservednon-reservednon-reserved
CLOBnon-reservednon-reservedreserved
CLOSEnon-reservednon-reservedreserved
CLUSTERnon-reservednon-reservednon-reserved
CLUSTEREDnon-reservednon-reservednon-reserved
COALESCEnon-reservednon-reservedreserved
CODEGENnon-reservednon-reservednon-reserved
COLLATEreservednon-reservedreserved
COLLATIONnon-reservednon-reservednon-reserved
COLLECTnon-reservednon-reservedreserved
COLLECTIONnon-reservednon-reservednon-reserved
COLUMNreservednon-reservedreserved
COLUMNSnon-reservednon-reservednon-reserved
COMMENTnon-reservednon-reservednon-reserved
COMMITnon-reservednon-reservedreserved
COMPACTnon-reservednon-reservednon-reserved
COMPACTIONSnon-reservednon-reservednon-reserved
COMPUTEnon-reservednon-reservednon-reserved
CONCATENATEnon-reservednon-reservednon-reserved
CONDITIONnon-reservednon-reservedreserved
CONNECTnon-reservednon-reservednon-reserved
CONNECTIONnon-reservednon-reservednon-reserved
CONSTRAINTreservednon-reservedreserved
CONSTRAINTSnon-reservednon-reservednon-reserved
CONSTRUCTORnon-reservednon-reservednon-reserved
CONTAINSnon-reservednon-reservednon-reserved
CONTINUEnon-reservednon-reservednon-reserved
CONVERTnon-reservednon-reservedreserved
COPYnon-reservednon-reservednon-reserved
CORRnon-reservednon-reservedreserved
CORRESPONDINGnon-reservednon-reservedreserved
COSnon-reservednon-reservednon-reserved
COSHnon-reservednon-reservednon-reserved
COSTnon-reservednon-reservednon-reserved
COUNTnon-reservednon-reservedreserved
COVAR_POPnon-reservednon-reservedreserved
COVAR_SAMPnon-reservednon-reservedreserved
CREATEreservednon-reservedreserved
CROSSreservednon-reservedreserved
CUBEnon-reservednon-reservedreserved
CUME_DISTnon-reservednon-reservedreserved
CURRENTnon-reservednon-reservedreserved
CURRENT_CATALOGnon-reservednon-reservedreserved
CURRENT_DATEreservednon-reservedreserved
CURRENT_DEFAULT_TRANSFORM_GROUPnon-reservednon-reservedreserved
CURRENT_PATHnon-reservednon-reservedreserved
CURRENT_ROLEnon-reservednon-reservedreserved
CURRENT_ROWnon-reservednon-reservedreserved
CURRENT_SCHEMAnon-reservednon-reservedreserved
CURRENT_TIMEreservednon-reservedreserved
CURRENT_TIMESTAMPreservednon-reservedreserved
CURRENT_TRANSFORM_GROUP_FOR_TYPEnon-reservednon-reservedreserved
CURRENT_USERreservednon-reservedreserved
CURSORnon-reservednon-reservedreserved
CYCLEnon-reservednon-reservedreserved
DATAnon-reservednon-reservednon-reserved
DATABASEnon-reservednon-reservednon-reserved
DATABASESnon-reservednon-reservednon-reserved
DATEnon-reservednon-reservedreserved
DAYnon-reservednon-reservedreserved
DBPROPERTIESnon-reservednon-reservednon-reserved
DEALLOCATEnon-reservednon-reservedreserved
DECnon-reservednon-reservedreserved
DECFLOATnon-reservednon-reservednon-reserved
DECIMALnon-reservednon-reservedreserved
DECLAREnon-reservednon-reservedreserved
DEFAULTnon-reservednon-reservedreserved
DEFERRABLEnon-reservednon-reservednon-reserved
DEFERREDnon-reservednon-reservednon-reserved
DEFINEnon-reservednon-reservednon-reserved
DEFINEDnon-reservednon-reservednon-reserved
DELETEnon-reservednon-reservedreserved
DELIMITEDnon-reservednon-reservednon-reserved
DENSE_RANKnon-reservednon-reservedreserved
DEPTHnon-reservednon-reservednon-reserved
DEREFnon-reservednon-reservedreserved
DESCnon-reservednon-reservednon-reserved
DESCRIBEnon-reservednon-reservedreserved
DESCRIPTORnon-reservednon-reservednon-reserved
DETERMINISTICnon-reservednon-reservedreserved
DFSnon-reservednon-reservednon-reserved
DIAGNOSTICSnon-reservednon-reservednon-reserved
DIRECTORIESnon-reservednon-reservednon-reserved
DIRECTORYnon-reservednon-reservednon-reserved
DISCONNECTnon-reservednon-reservedreserved
DISTINCTreservednon-reservedreserved
DISTRIBUTEnon-reservednon-reservednon-reserved
DIVnon-reservednon-reservednon-reserved
DOnon-reservednon-reservedreserved
DOMAINnon-reservednon-reservednon-reserved
DOUBLEnon-reservednon-reservedreserved
DROPnon-reservednon-reservedreserved
DYNAMICnon-reservednon-reservedreserved
EACHnon-reservednon-reservedreserved
ELEMENTnon-reservednon-reservedreserved
ELSEreservednon-reservedreserved
ELSEIFnon-reservednon-reservedreserved
EMPTYnon-reservednon-reservednon-reserved
ENDreservednon-reservedreserved
END_FRAMEnon-reservednon-reservedreserved
END_PARTITIONnon-reservednon-reservedreserved
EQUALSnon-reservednon-reservednon-reserved
ESCAPEnon-reservednon-reservedreserved
ESCAPEDnon-reservednon-reservednon-reserved
EVERYnon-reservednon-reservedreserved
EXCEPTreservednon-reservedreserved
EXCEPTIONnon-reservednon-reservednon-reserved
EXCHANGEnon-reservednon-reservednon-reserved
EXECnon-reservednon-reservedreserved
EXECUTEnon-reservednon-reservedreserved
EXISTSnon-reservednon-reservedreserved
EXITnon-reservednon-reservednon-reserved
EXPnon-reservednon-reservednon-reserved
EXPLAINnon-reservednon-reservednon-reserved
EXPORTnon-reservednon-reservednon-reserved
EXTENDEDnon-reservednon-reservednon-reserved
EXTERNALnon-reservednon-reservedreserved
EXTRACTnon-reservednon-reservedreserved
FALSEreservednon-reservedreserved
FETCHreservednon-reservedreserved
FIELDSnon-reservednon-reservednon-reserved
FILEFORMATnon-reservednon-reservednon-reserved
FILTERnon-reservednon-reservedreserved
FIRSTnon-reservednon-reservednon-reserved
FIRST_VALUEnon-reservednon-reservedreserved
FLOATnon-reservednon-reservedreserved
FOLLOWINGnon-reservednon-reservednon-reserved
FORreservednon-reservedreserved
FOREIGNreservednon-reservedreserved
FORMATnon-reservednon-reservednon-reserved
FORMATTEDnon-reservednon-reservednon-reserved
FOUNDnon-reservednon-reservednon-reserved
FRAME_ROWnon-reservednon-reservedreserved
FREEnon-reservednon-reservedreserved
FROMreservednon-reservedreserved
FULLreservednon-reservedreserved
FUNCTIONnon-reservednon-reservedreserved
FUNCTIONSnon-reservednon-reservednon-reserved
FUSIONnon-reservednon-reservednon-reserved
GENERALnon-reservednon-reservednon-reserved
GETnon-reservednon-reservedreserved
GLOBALnon-reservednon-reservedreserved
GOnon-reservednon-reservednon-reserved
GOTOnon-reservednon-reservednon-reserved
GRANTreservednon-reservedreserved
GROUPreservednon-reservedreserved
GROUPINGnon-reservednon-reservedreserved
GROUPSnon-reservednon-reservedreserved
HANDLERnon-reservednon-reservedreserved
HAVINGreservednon-reservedreserved
HOLDnon-reservednon-reservedreserved
HOURnon-reservednon-reservedreserved
IDENTITYnon-reservednon-reservedreserved
IFnon-reservednon-reservedreserved
IGNOREnon-reservednon-reservednon-reserved
IMMEDIATEnon-reservednon-reservednon-reserved
IMPORTnon-reservednon-reservednon-reserved
INreservednon-reservedreserved
INDICATORnon-reservednon-reservedreserved
INDEXnon-reservednon-reservednon-reserved
INDEXESnon-reservednon-reservednon-reserved
INITIALnon-reservednon-reservednon-reserved
INITIALLYnon-reservednon-reservednon-reserved
INNERreservednon-reservedreserved
INOUTnon-reservednon-reservedreserved
INPATHnon-reservednon-reservednon-reserved
INPUTnon-reservednon-reservednon-reserved
INPUTFORMATnon-reservednon-reservednon-reserved
INSENSITIVEnon-reservednon-reservedreserved
INSERTnon-reservednon-reservedreserved
INTnon-reservednon-reservedreserved
INTEGERnon-reservednon-reservedreserved
INTERSECTreservednon-reservedreserved
INTERSECTIONnon-reservednon-reservedreserved
INTERVALnon-reservednon-reservedreserved
INTOreservednon-reservedreserved
ISreservednon-reservedreserved
ISOLATIONnon-reservednon-reservednon-reserved
ITEMSnon-reservednon-reservednon-reserved
ITERATEnon-reservednon-reservedreserved
JOINreservednon-reservedreserved
JSON_ARRAYnon-reservednon-reservednon-reserved
JSON_ARRAYAGGnon-reservednon-reservednon-reserved
JSON_EXISTSnon-reservednon-reservednon-reserved
JSON_OBJECTnon-reservednon-reservednon-reserved
JSON_OBJECTAGGnon-reservednon-reservednon-reserved
JSON_QUERYnon-reservednon-reservednon-reserved
JSON_TABLEnon-reservednon-reservednon-reserved
JSON_TABLE_PRIMITIVEnon-reservednon-reservednon-reserved
JSON_VALUEnon-reservednon-reservednon-reserved
KEYnon-reservednon-reservednon-reserved
KEYSnon-reservednon-reservednon-reserved
LAGnon-reservednon-reservednon-reserved
LANGUAGEnon-reservednon-reservedreserved
LARGEnon-reservednon-reservedreserved
LASTnon-reservednon-reservednon-reserved
LAST_VALUEnon-reservednon-reservedreserved
LATERALnon-reservednon-reservedreserved
LAZYnon-reservednon-reservednon-reserved
LEADnon-reservednon-reservedreserved
LEADINGreservednon-reservedreserved
LEAVEnon-reservednon-reservedreserved
LEFTreservednon-reservedreserved
LEVELnon-reservednon-reservednon-reserved
LIKEnon-reservednon-reservedreserved
LIKE_REGEXnon-reservednon-reservedreserved
LIMITnon-reservednon-reservednon-reserved
LINESnon-reservednon-reservednon-reserved
LISTnon-reservednon-reservednon-reserved
LISTAGGnon-reservednon-reservednon-reserved
LNnon-reservednon-reservedreserved
LOADnon-reservednon-reservednon-reserved
LOCALnon-reservednon-reservedreserved
LOCALTIMEnon-reservednon-reservedreserved
LOCALTIMESTAMPnon-reservednon-reservedreserved
LOCATIONnon-reservednon-reservednon-reserved
LOCATORnon-reservednon-reservednon-reserved
LOCKnon-reservednon-reservednon-reserved
LOCKSnon-reservednon-reservednon-reserved
LOGnon-reservednon-reservednon-reserved
LOG10non-reservednon-reservednon-reserved
LOGICALnon-reservednon-reservednon-reserved
LOOPnon-reservednon-reservedreserved
LOWERnon-reservednon-reservedreserved
MACROnon-reservednon-reservednon-reserved
MAPnon-reservednon-reservednon-reserved
MATCHnon-reservednon-reservedreserved
MATCH_NUMBERnon-reservednon-reservednon-reserved
MATCH_RECOGNIZEnon-reservednon-reservednon-reserved
MATCHESnon-reservednon-reservednon-reserved
MAXnon-reservednon-reservedreserved
MEMBERnon-reservednon-reservedreserved
MERGEnon-reservednon-reservedreserved
METHODnon-reservednon-reservedreserved
MINnon-reservednon-reservedreserved
MINUTEnon-reservednon-reservedreserved
MODnon-reservednon-reservedreserved
MODIFIESnon-reservednon-reservedreserved
MODULEnon-reservednon-reservedreserved
MONTHnon-reservednon-reservedreserved
MSCKnon-reservednon-reservednon-reserved
MULTISETnon-reservednon-reservedreserved
NAMESnon-reservednon-reservednon-reserved
NATIONALnon-reservednon-reservedreserved
NATURALreservednon-reservedreserved
NCHARnon-reservednon-reservedreserved
NCLOBnon-reservednon-reservedreserved
NEWnon-reservednon-reservedreserved
NEXTnon-reservednon-reservednon-reserved
NOnon-reservednon-reservedreserved
NONEnon-reservednon-reservedreserved
NORMALIZEnon-reservednon-reservedreserved
NOTreservednon-reservedreserved
NTH_VALUEnon-reservednon-reservedreserved
NTILEnon-reservednon-reservedreserved
NULLreservednon-reservedreserved
NULLSnon-reservednon-reservednon-reserved
NULLIFnon-reservednon-reservedreserved
NUMERICnon-reservednon-reservedreserved
OBJECTnon-reservednon-reservednon-reserved
OCCURRENCES_REGEXnon-reservednon-reservednon-reserved
OCTET_LENGTHnon-reservednon-reservedreserved
OFnon-reservednon-reservedreserved
OFFSETnon-reservednon-reservedreserved
OLDnon-reservednon-reservedreserved
OMITnon-reservednon-reservednon-reserved
ONreservednon-reservedreserved
ONEnon-reservednon-reservednon-reserved
ONLYreservednon-reservedreserved
OPENnon-reservednon-reservedreserved
OPTIONnon-reservednon-reservednon-reserved
OPTIONSnon-reservednon-reservednon-reserved
ORreservednon-reservedreserved
ORDERreservednon-reservedreserved
ORDINALITYnon-reservednon-reservednon-reserved
OUTnon-reservednon-reservedreserved
OUTERreservednon-reservedreserved
OUTPUTnon-reservednon-reservednon-reserved
OUTPUTFORMATnon-reservednon-reservednon-reserved
OVERnon-reservednon-reservednon-reserved
OVERLAPSreservednon-reservedreserved
OVERLAYnon-reservednon-reservedreserved
OVERWRITEnon-reservednon-reservednon-reserved
PADnon-reservednon-reservednon-reserved
PARAMETERnon-reservednon-reservedreserved
PARTIALnon-reservednon-reservednon-reserved
PARTITIONnon-reservednon-reservedreserved
PARTITIONEDnon-reservednon-reservednon-reserved
PARTITIONSnon-reservednon-reservednon-reserved
PATHnon-reservednon-reservednon-reserved
PATTERNnon-reservednon-reservednon-reserved
PERnon-reservednon-reservednon-reserved
PERCENTnon-reservednon-reservedreserved
PERCENT_RANKnon-reservednon-reservedreserved
PERCENTILE_CONTnon-reservednon-reservedreserved
PERCENTILE_DISCnon-reservednon-reservedreserved
PERCENTLITnon-reservednon-reservednon-reserved
PERIODnon-reservednon-reservedreserved
PIVOTnon-reservednon-reservednon-reserved
PORTIONnon-reservednon-reservedreserved
POSITIONnon-reservednon-reservedreserved
POSITION_REGEXnon-reservednon-reservedreserved
POWERnon-reservednon-reservedreserved
PRECEDESnon-reservednon-reservedreserved
PRECEDINGnon-reservednon-reservednon-reserved
PRECISIONnon-reservednon-reservedreserved
PREPAREnon-reservednon-reservedreserved
PRESERVEnon-reservednon-reservednon-reserved
PRIMARYreservednon-reservedreserved
PRINCIPALSnon-reservednon-reservednon-reserved
PRIORnon-reservednon-reservednon-reserved
PRIVILEGESnon-reservednon-reservednon-reserved
PROCEDUREnon-reservednon-reservedreserved
PTFnon-reservednon-reservednon-reserved
PUBLICnon-reservednon-reservednon-reserved
PURGEnon-reservednon-reservednon-reserved
RANGEnon-reservednon-reservedreserved
RANKnon-reservednon-reservedreserved
READnon-reservednon-reservednon-reserved
READSnon-reservednon-reservedreserved
REALnon-reservednon-reservedreserved
RECORDREADERnon-reservednon-reservednon-reserved
RECORDWRITERnon-reservednon-reservednon-reserved
RECURSIVEnon-reservednon-reservedreserved
RECOVERnon-reservednon-reservednon-reserved
REDUCEnon-reservednon-reservednon-reserved
REFnon-reservednon-reservedreserved
REFERENCESreservednon-reservedreserved
REFERENCINGnon-reservednon-reservedreserved
REFRESHnon-reservednon-reservednon-reserved
REGR_AVGXnon-reservednon-reservedreserved
REGR_AVGYnon-reservednon-reservedreserved
REGR_COUNTnon-reservednon-reservedreserved
REGR_INTERCEPTnon-reservednon-reservedreserved
REGR_R2non-reservednon-reservedreserved
REGR_SLOPEnon-reservednon-reservedreserved
REGR_SXXnon-reservednon-reservedreserved
REGR_SXYnon-reservednon-reservedreserved
REGR_SYYnon-reservednon-reservedreserved
RELATIVEnon-reservednon-reservednon-reserved
RELEASEnon-reservednon-reservedreserved
RENAMEnon-reservednon-reservednon-reserved
REPAIRnon-reservednon-reservednon-reserved
REPEATnon-reservednon-reservedreserved
REPLACEnon-reservednon-reservednon-reserved
RESETnon-reservednon-reservednon-reserved
RESIGNALnon-reservednon-reservedreserved
RESTRICTnon-reservednon-reservednon-reserved
RESULTnon-reservednon-reservedreserved
RETURNnon-reservednon-reservedreserved
RETURNSnon-reservednon-reservedreserved
REVOKEnon-reservednon-reservedreserved
RIGHTreservednon-reservedreserved
RLIKEnon-reservednon-reservednon-reserved
ROLEnon-reservednon-reservednon-reserved
ROLESnon-reservednon-reservednon-reserved
ROLLBACKnon-reservednon-reservedreserved
ROLLUPnon-reservednon-reservedreserved
ROUTINEnon-reservednon-reservednon-reserved
ROWnon-reservednon-reservedreserved
ROW_NUMBERnon-reservednon-reservedreserved
ROWSnon-reservednon-reservedreserved
RUNNINGnon-reservednon-reservednon-reserved
SAVEPOINTnon-reservednon-reservedreserved
SCHEMAnon-reservednon-reservednon-reserved
SCOPEnon-reservednon-reservedreserved
SCROLLnon-reservednon-reservedreserved
SEARCHnon-reservednon-reservedreserved
SECONDnon-reservednon-reservedreserved
SECTIONnon-reservednon-reservednon-reserved
SEEKnon-reservednon-reservednon-reserved
SELECTreservednon-reservedreserved
SENSITIVEnon-reservednon-reservedreserved
SEPARATEDnon-reservednon-reservednon-reserved
SERDEnon-reservednon-reservednon-reserved
SERDEPROPERTIESnon-reservednon-reservednon-reserved
SESSIONnon-reservednon-reservednon-reserved
SESSION_USERreservednon-reservedreserved
SETnon-reservednon-reservedreserved
SETSnon-reservednon-reservednon-reserved
SHOWnon-reservednon-reservednon-reserved
SIGNALnon-reservednon-reservedreserved
SIMILARnon-reservednon-reservedreserved
SINnon-reservednon-reservednon-reserved
SINHnon-reservednon-reservednon-reserved
SIZEnon-reservednon-reservednon-reserved
SKIPnon-reservednon-reservednon-reserved
SKEWEDnon-reservednon-reservednon-reserved
SMALLINTnon-reservednon-reservedreserved
SOMEreservednon-reservedreserved
SORTnon-reservednon-reservednon-reserved
SORTEDnon-reservednon-reservednon-reserved
SPACEnon-reservednon-reservednon-reserved
SPECIFICnon-reservednon-reservedreserved
SPECIFICTYPEnon-reservednon-reservedreserved
SQLnon-reservednon-reservedreserved
SQLCODEnon-reservednon-reservednon-reserved
SQLERRORnon-reservednon-reservednon-reserved
SQLEXCEPTIONnon-reservednon-reservedreserved
SQLSTATEnon-reservednon-reservedreserved
SQLWARNINGnon-reservednon-reservedreserved
SQRTnon-reservednon-reservedreserved
STARTnon-reservednon-reservedreserved
STATEnon-reservednon-reservednon-reserved
STATICnon-reservednon-reservedreserved
STATISTICSnon-reservednon-reservednon-reserved
STDDEV_POPnon-reservednon-reservedreserved
STDDEV_SAMPnon-reservednon-reservedreserved
STOREDnon-reservednon-reservednon-reserved
STRATIFYnon-reservednon-reservednon-reserved
STRUCTnon-reservednon-reservednon-reserved
SUBMULTISETnon-reservednon-reservedreserved
SUBSETnon-reservednon-reservednon-reserved
SUBSTRINGnon-reservednon-reservedreserved
SUBSTRING_REGEXnon-reservednon-reservedreserved
SUCCEEDSnon-reservednon-reservedreserved
SUMnon-reservednon-reservedreserved
SYMMETRICnon-reservednon-reservedreserved
SYSTEMnon-reservednon-reservedreserved
SYSTEM_TIMEnon-reservednon-reservedreserved
SYSTEM_USERnon-reservednon-reservedreserved
TABLEreservednon-reservedreserved
TABLESnon-reservednon-reservednon-reserved
TABLESAMPLEnon-reservednon-reservedreserved
TANnon-reservednon-reservednon-reserved
TANHnon-reservednon-reservednon-reserved
TBLPROPERTIESnon-reservednon-reservednon-reserved
TEMPORARYnon-reservednon-reservednon-reserved
TERMINATEDnon-reservednon-reservednon-reserved
THENreservednon-reservedreserved
TIMEnon-reservednon-reservedreserved
TIMESTAMPnon-reservednon-reservedreserved
TIMEZONE_HOURnon-reservednon-reservedreserved
TIMEZONE_MINUTEnon-reservednon-reservedreserved
TOreservednon-reservedreserved
TOUCHnon-reservednon-reservednon-reserved
TRAILINGreservednon-reservedreserved
TRANSACTIONnon-reservednon-reservednon-reserved
TRANSACTIONSnon-reservednon-reservednon-reserved
TRANSFORMnon-reservednon-reservednon-reserved
TRANSLATEnon-reservednon-reservedreserved
TRANSLATE_REGEXnon-reservednon-reservedreserved
TRANSLATIONnon-reservednon-reservedreserved
TREATnon-reservednon-reservedreserved
TRIGGERnon-reservednon-reservedreserved
TRIMnon-reservednon-reservedreserved
TRIM_ARRAYnon-reservednon-reservedreserved
TRUEnon-reservednon-reservedreserved
TRUNCATEnon-reservednon-reservedreserved
UESCAPEnon-reservednon-reservedreserved
UNARCHIVEnon-reservednon-reservednon-reserved
UNBOUNDEDnon-reservednon-reservednon-reserved
UNCACHEnon-reservednon-reservednon-reserved
UNDERnon-reservednon-reservednon-reserved
UNDOnon-reservednon-reservedreserved
UNIONreservednon-reservedreserved
UNIQUEreservednon-reservedreserved
UNKNOWNnon-reservednon-reservedreserved
UNLOCKnon-reservednon-reservednon-reserved
UNNESTnon-reservednon-reservedreserved
UNSETnon-reservednon-reservednon-reserved
UNTILnon-reservednon-reservedreserved
UPDATEnon-reservednon-reservedreserved
UPPERnon-reservednon-reservedreserved
USAGEnon-reservednon-reservednon-reserved
USEnon-reservednon-reservednon-reserved
USERreservednon-reservedreserved
USINGreservedreservedreserved
VALUEnon-reservednon-reservedreserved
VALUESnon-reservednon-reservedreserved
VALUE_OFnon-reservednon-reservedreserved
VAR_POPnon-reservednon-reservedreserved
VAR_SAMPnon-reservednon-reservedreserved
VARBINARYnon-reservednon-reservedreserved
VARCHARnon-reservednon-reservedreserved
VARYINGnon-reservednon-reservedreserved
VERSIONINGnon-reservednon-reservedreserved
VIEWnon-reservednon-reservednon-reserved
WHENreservednon-reservedreserved
WHENEVERnon-reservednon-reservedreserved
WHEREreservednon-reservedreserved
WHILEnon-reservednon-reservedreserved
WIDTH_BUCKETnon-reservednon-reservedreserved
WINDOWnon-reservednon-reservedreserved
WITHreservednon-reservedreserved
WITHINnon-reservednon-reservedreserved
WITHOUTnon-reservednon-reservedreserved
WORKnon-reservednon-reservednon-reserved
WRITEnon-reservednon-reservednon-reserved
YEARnon-reservednon-reservedreserved
ZONEnon-reservednon-reservednon-reserved
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 19b4432e08dea..d9548dc00c5f5 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -782,22 +782,22 @@ ansiNonReserved nonReserved : ADD | AFTER | ALL | ALTER | ANALYZE | AND | ANY | ARCHIVE | ARRAY | AS | ASC | AT | AUTHORIZATION | BETWEEN - | BOTH | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHECK | COLLATE | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN - | COLLECTION | COLUMN | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST - | CREATE | CUBE | CURRENT | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DATA | DATABASE - | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY - | DISTINCT | DISTRIBUTE | DIV | DROP | ELSE | END | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT | EXTENDED - | EXTERNAL | EXTRACT | FALSE | FETCH | FOREIGN | FIELDS | FILEFORMAT | FIRST | FOLLOWING | FOR | FORMAT + | BOTH | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHANGE | CHECK | CLEAR | CLUSTER | CLUSTERED + | CODEGEN | COLLATE | COLLECTION | COLUMN | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE + | CONCATENATE | CONSTRAINT | COST | CREATE | CUBE | CURRENT | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER + | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES + | DIRECTORY | DISTINCT | DISTRIBUTE | DIV | DROP | ELSE | END | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT + | EXTENDED | EXTERNAL | EXTRACT | FALSE | FETCH | FIELDS | FILEFORMAT | FIRST | FOLLOWING | FOR | FOREIGN | FORMAT | FORMATTED | FROM | FUNCTION | FUNCTIONS | GLOBAL | GRANT | GROUP | GROUPING | HAVING | IF | IGNORE | IMPORT - | IN | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | INTO | IS |LIKE | ITEMS | KEYS | LAST - | LATERAL | LAZY | LEADING | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO + | IN | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | INTO | IS | ITEMS | KEYS | LAST | LATERAL + | LAZY | LEADING | LIKE | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO | MAP | MSCK | NO | NOT | NULL | NULLS | OF | ONLY | OPTION | OPTIONS | OR | ORDER | OUT | OUTER | OUTPUTFORMAT | OVER | OVERLAPS | OVERWRITE | PARTITION | PARTITIONED | PARTITIONS | PERCENTLIT | PIVOT | POSITION | PRECEDING - | PRIMARY | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFERENCES | REFRESH | RENAME | REPAIR - | REPLACE | RESET | RESTRICT | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SELECT - | SEPARATED | SERDE | SERDEPROPERTIES | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED | START | STATISTICS - | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | THEN - | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED + | PRIMARY | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFERENCES | REFRESH + | RENAME | REPAIR | REPLACE | RESET | RESTRICT | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS + | SELECT | SEPARATED | SERDE | SERDEPROPERTIES | SESSION_USER | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED | START + | STATISTICS | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED + | THEN | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index be6af24999e8e..28b7108a94235 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -59,10 +59,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "natural", "union", "intersect", "except", "database", "on", "join", "cross", "select", "from", "where", "having", "from", "to", "table", "with", "not") - val otherNonReservedKeywords = Seq( - "check", "collate", "overlaps", "only", "primary", "references", "unique") - - val ansiReservedKeywords = Seq("all", "and", "any", "as", "authorization", "both", "case", + val reservedKeywordsInAnsi = Seq("all", "and", "any", "as", "authorization", "both", "case", "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", @@ -71,9 +68,9 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "when", "where", "with") - val ansiNonReservedKeywords = Seq("abs", "absolute", "acos", "action", "add", "after", "allocate", - "alter", "analyze", "are", "archive", "array", "array_agg", "array_max_cardinality", "asc", - "asensitive", "asin", "assertion", "asymmetric", "at", "atan", "atomic", "avg", "before", + val nonReservedKeywordsInAnsi = Seq("abs", "absolute", "acos", "action", "add", "after", + "allocate", "alter", "analyze", "are", "archive", "array", "array_agg", "array_max_cardinality", + "asc", "asensitive", "asin", "assertion", "asymmetric", "at", "atan", "atomic", "avg", "before", "begin", "begin_frame", "begin_partition", "between", "bigint", "binary", "bit", "bit_length", "blob", "boolean", "breadth", "bucket", "buckets", "by", "cache", "call", "called", "cardinality", "cascade", "cascaded", "catalog", "ceil", "ceiling", "change", "char", @@ -134,6 +131,74 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "work", "write", "year", "zone" ) + val nonReservedKeywordsInNonAnsi = Seq("abs", "absolute", "acos", "action", "add", "after", + "all", "allocate", "alter", "analyze", "and", "any", "archive", "are", "array", "array_agg", + "array_max_cardinality", "as", "asc", "asensitive", "asin", "assertion", "asymmetric", "at", + "atan", "atomic", "authorization", "avg", "before", "begin", "begin_frame", "begin_partition", + "between", "bigint", "binary", "bit", "bit_length", "blob", "boolean", "both", "breadth", + "bucket", "buckets", "by", "cache", "call", "called", "cardinality", "cascade", "cascaded", + "case", "cast", "catalog", "ceil", "ceiling", "change", "char", "char_length", "character", + "character_length", "check", "classifier", "clear", "clob", "close", "cluster", "clustered", + "coalesce", "codegen", "collate", "collation", "collect", "collection", "column", "columns", + "comment", "commit", "compact", "compactions", "compute", "concatenate", "condition", "connect", + "connection", "constraint", "constraints", "constructor", "contains", "continue", "convert", + "copy", "corr", "corresponding", "cos", "cosh", "cost", "count", "covar_pop", "covar_samp", + "create", "cross", "cube", "cume_dist", "current", "current_catalog", "current_date", + "current_default_transform_group", "current_path", "current_role", "current_row", + "current_schema", "current_time", "current_timestamp", "current_transform_group_for_type", + "current_user", "cursor", "cycle", "data", "database", "databases", "date", "day", + "dbproperties", "deallocate", "dec", "decfloat", "decimal", "declare", "default", "deferrable", + "deferred", "define", "defined", "delete", "delimited", "dense_rank", "depth", "deref", "desc", + "describe", "descriptor", "deterministic", "dfs", "diagnostics", "directories", "directory", + "disconnect", "distinct", "distribute", "div", "do", "domain", "double", "drop", "dynamic", + "each", "element", "else", "elseif", "empty", "end", "end_frame", "end_partition", "equals", + "escape", "escaped", "every", "except", "exception", "exchange", "exec", "execute", "exists", + "exit", "exp", "explain", "export", "extended", "external", "extract", "false", "fetch", + "fields", "fileformat", "filter", "first", "first_value", "float", "following", "for", + "foreign", "format", "formatted", "found", "frame_row", "free", "from", "full", "function", + "functions", "fusion", "general", "get", "global", "go", "goto", "grant", "group", "grouping", + "groups", "handler", "having", "hold", "hour", "identity", "if", "ignore", "immediate", + "import", "in", "index", "indexes", "indicator", "initial", "initially", "inner", "inout", + "inpath", "input", "inputformat", "insensitive", "insert", "int", "integer", "intersect", + "intersection", "interval", "into", "is", "isolation", "items", "iterate", "join", "json_array", + "json_arrayagg", "json_exists", "json_object", "json_objectagg", "json_query", "json_table", + "json_table_primitive", "json_value", "key", "keys", "lag", "language", "large", "last", + "last_value", "lateral", "lazy", "lead", "leading", "leave", "left", "level", "like", + "like_regex", "limit", "lines", "list", "listagg", "ln", "load", "local", "localtime", + "localtimestamp", "location", "locator", "lock", "locks", "log", "log10", "logical", "loop", + "lower", "macro", "map", "match", "match_number", "match_recognize", "matches", "max", "member", + "merge", "method", "min", "minute", "mod", "modifies", "module", "month", "msck", "multiset", + "names", "national", "natural", "nchar", "nclob", "new", "next", "no", "none", "normalize", + "not", "nth_value", "ntile", "null", "nullif", "nulls", "numeric", "object", + "occurrences_regex", "octet_length", "of", "offset", "old", "omit", "on", "one", "only", "open", + "option", "options", "or", "order", "ordinality", "out", "outer", "output", "outputformat", + "over", "overlaps", "overlay", "overwrite", "pad", "parameter", "partial", "partition", + "partitioned", "partitions", "path", "pattern", "per", "percent", "percent_rank", + "percentile_cont", "percentile_disc", "percentlit", "period", "pivot", "portion", "power", + "precedes", "preceding", "precision", "prepare", "preserve", "primary", "principals", "prior", + "privileges", "procedure", "ptf", "public", "purge", "range", "rank", "read", "reads", "real", + "recordreader", "recordwriter", "recover", "recursive", "reduce", "ref", "references", + "referencing", "refresh", "regr_avgx", "regr_avgy", "regr_count", "regr_intercept", "regr_r2", + "regr_slope", "regr_sxx", "regr_sxy", "regr_syy", "relative", "release", "rename", "repair", + "repeat", "replace", "reset", "resignal", "restrict", "result", "return", "returns", "revoke", + "right", "rlike", "role", "roles", "rollback", "rollup", "routine", "row", "row_number", "rows", + "running", "savepoint", "schema", "scope", "scroll", "search", "second", "section", "seek", + "select", "sensitive", "separated", "serde", "serdeproperties", "session", "session_user", + "set", "sets", "show", "signal", "similar", "sin", "sinh", "size", "skewed", "skip", "smallint", + "some", "sort", "sorted", "space", "specific", "specifictype", "sql", "sqlcode", "sqlerror", + "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", "state", "static", "statistics", + "stddev_pop", "stddev_samp", "stored", "stratify", "struct", "submultiset", "subset", + "substring", "substring_regex", "succeeds", "sum", "symmetric", "system", "system_time", + "system_user", "table", "tables", "tablesample", "tan", "tanh", "tblproperties", "temporary", + "terminated", "then", "time", "timestamp", "timezone_hour", "timezone_minute", "to", "touch", + "trailing", "transaction", "transactions", "transform", "translate", "translate_regex", + "translation", "treat", "trigger", "trim", "trim_array", "true", "truncate", "uescape", + "unarchive", "unbounded", "uncache", "under", "undo", "union", "unique", "unknown", "unlock", + "unnest", "unset", "until", "update", "upper", "usage", "use", "user", "value", "value_of", + "values", "var_pop", "var_samp", "varbinary", "varchar", "varying", "versioning", "view", + "when", "whenever", "where", "while", "width_bucket", "window", "with", "within", "without", + "work", "write", "year", "zone") + test("table identifier") { // Regular names. assert(TableIdentifier("q") === parseTableIdentifier("q")) @@ -155,7 +220,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { test("table identifier - reserved/non-reserved keywords if ANSI mode enabled") { withSQLConf(SQLConf.ANSI_SQL_PARSER.key -> "true") { - ansiReservedKeywords.foreach { keyword => + reservedKeywordsInAnsi.foreach { keyword => val errMsg = intercept[ParseException] { parseTableIdentifier(keyword) }.getMessage @@ -164,7 +229,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) } - ansiNonReservedKeywords.foreach { keyword => + nonReservedKeywordsInAnsi.foreach { keyword => assert(TableIdentifier(keyword) === parseTableIdentifier(s"$keyword")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.$keyword")) } @@ -181,8 +246,12 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { } test("table identifier - non reserved keywords") { - // Hive keywords are allowed. - (hiveNonReservedKeywords ++ otherNonReservedKeywords).foreach { nonReserved => + nonReservedKeywordsInNonAnsi.foreach { nonReserved => + assert(TableIdentifier(nonReserved) === parseTableIdentifier(nonReserved)) + } + + // Additionally checks if Hive keywords can be allowed. + hiveNonReservedKeywords.foreach { nonReserved => assert(TableIdentifier(nonReserved) === parseTableIdentifier(nonReserved)) } } From 57fa8ed6a3dd55ec33592c6532e40eb774fccf12 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Sun, 17 Feb 2019 15:16:13 +0900 Subject: [PATCH 07/18] Brush up test code --- .../parser/TableIdentifierParserSuite.scala | 102 ++++-------------- 1 file changed, 22 insertions(+), 80 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index 28b7108a94235..289657d2f08dd 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -59,79 +59,8 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "natural", "union", "intersect", "except", "database", "on", "join", "cross", "select", "from", "where", "having", "from", "to", "table", "with", "not") - val reservedKeywordsInAnsi = Seq("all", "and", "any", "as", "authorization", "both", "case", - "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", - "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", - "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", - "intersect", "into", "is", "join", "leading", "left", "natural", "not", "null", "on", "only", - "or", "order", "outer", "overlaps", "primary", "references", "right", "select", "session_user", - "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "when", - "where", "with") - - val nonReservedKeywordsInAnsi = Seq("abs", "absolute", "acos", "action", "add", "after", - "allocate", "alter", "analyze", "are", "archive", "array", "array_agg", "array_max_cardinality", - "asc", "asensitive", "asin", "assertion", "asymmetric", "at", "atan", "atomic", "avg", "before", - "begin", "begin_frame", "begin_partition", "between", "bigint", "binary", "bit", "bit_length", - "blob", "boolean", "breadth", "bucket", "buckets", "by", "cache", "call", "called", - "cardinality", "cascade", "cascaded", "catalog", "ceil", "ceiling", "change", "char", - "char_length", "character", "character_length", "classifier", "clear", "clob", "close", - "cluster", "clustered", "coalesce", "codegen", "collation", "collect", "collection", - "columns", "comment", "commit", "compact", "compactions", "compute", "concatenate", "condition", - "connect", "connection", "constraints", "constructor", "contains", "continue", "convert", - "copy", "corr", "corresponding", "cos", "cosh", "cost", "count", "covar_pop", "covar_samp", - "cube", "cume_dist", "current", "current_catalog", "current_default_transform_group", - "current_path", "current_role", "current_row", "current_schema", - "current_transform_group_for_type", "cursor", "cycle", "data", "database", "databases", "date", - "day", "dbproperties", "deallocate", "dec", "decfloat", "decimal", "declare", "default", - "deferrable", "deferred", "define", "defined", "delete", "delimited", "dense_rank", "depth", - "deref", "desc", "describe", "descriptor", "deterministic", "dfs", "diagnostics", - "directories", "directory", "disconnect", "distribute", "div", "do", "domain", "double", - "drop", "dynamic", "each", "element", "elseif", "empty", "end_frame", "end_partition", - "equals", "escape", "escaped", "every", "exception", "exchange", "exec", "execute", - "exists", "exit", "exp", "explain", "export", "extended", "external", "extract", "fields", - "fileformat", "filter", "first", "first_value", "float", "following", "format", "formatted", - "found", "frame_row", "free", "function", "functions", "fusion", "general", "get", "global", - "go", "goto", "grouping", "groups", "handler", "hold", "hour", "identity", "if", "ignore", - "immediate", "import", "indicator", "index", "indexes", "initial", "initially", "inout", - "inpath", "input", "inputformat", "insensitive", "insert", "int", "integer", "intersection", - "interval", "isolation", "items", "iterate", "json_array", "json_arrayagg", "json_exists", - "json_object", "json_objectagg", "json_query", "json_table", "json_table_primitive", - "json_value", "key", "keys", "lag", "language", "large", "last", "last_value", "lateral", - "lazy", "lead", "leave", "level", "like", "like_regex", "limit", "lines", "list", "listagg", - "ln", "load", "local", "localtime", "localtimestamp", "location", "locator", "lock", "locks", - "log", "log10", "logical", "loop", "lower", "macro", "map", "match", "match_number", - "match_recognize", "matches", "max", "member", "merge", "method", "min", "minute", "mod", - "modifies", "module", "month", "msck", "multiset", "names", "national", "nchar", "nclob", - "new", "next", "no", "none", "normalize", "nth_value", "ntile", "nulls", "nullif", "numeric", - "object", "occurrences_regex", "octet_length", "of", "offset", "old", "omit", "one", "open", - "option", "options", "ordinality", "out", "output", "outputformat", "over", "overlay", - "overwrite", "pad", "parameter", "partial", "partition", "partitioned", "partitions", "path", - "pattern", "per", "percent", "percent_rank", "percentile_cont", "percentile_disc", "percentlit", - "period", "pivot", "portion", "power", "precedes", "preceding", "precision", "prepare", - "preserve", "principals", "prior", "privileges", "procedure", "ptf", "public", "purge", "range", - "rank", "read", "reads", "real", "recordreader", "recordwriter", "recursive", "recover", - "reduce", "ref", "referencing", "refresh", "regr_avgx", "regr_avgy", "regr_count", - "regr_intercept", "regr_r2", "regr_slope", "regr_sxx", "regr_sxy", "regr_syy", "relative", - "release", "rename", "repair", "repeat", "replace", "reset", "resignal", "restrict", "result", - "return", "returns", "revoke", "rlike", "role", "roles", "rollback", "rollup", "routine", - "row", "row_number", "rows", "running", "savepoint", "schema", "scope", "scroll", "search", - "second", "section", "seek", "sensitive", "separated", "serde", "serdeproperties", "session", - "set", "sets", "show", "signal", "similar", "sin", "sinh", "size", "skip", "skewed", "smallint", - "sort", "sorted", "space", "specific", "specifictype", "sql", "sqlcode", "sqlerror", - "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", "state", "static", "statistics", - "stddev_pop", "stddev_samp", "stored", "stratify", "struct", "submultiset", "subset", - "substring", "substring_regex", "succeeds", "sum", "symmetric", "system", "system_time", - "system_user", "tables", "tablesample", "tan", "tanh", "tblproperties", "temporary", - "terminated", "time", "timestamp", "timezone_hour", "timezone_minute", "touch", "transaction", - "transactions", "transform", "translate", "translate_regex", "translation", "treat", "trigger", - "trim", "trim_array", "true", "truncate", "uescape", "unarchive", "unbounded", "uncache", - "under", "undo", "unknown", "unlock", "unnest", "unset", "until", "update", "upper", "usage", - "use", "value", "values", "value_of", "var_pop", "var_samp", "varbinary", "varchar", "varying", - "versioning", "view", "whenever", "while", "width_bucket", "window", "within", "without", - "work", "write", "year", "zone" - ) - - val nonReservedKeywordsInNonAnsi = Seq("abs", "absolute", "acos", "action", "add", "after", + // All the keywords in `docs/sql-reserved-and-non-reserved-key-words.md` are listed below: + val allCandidateKeywords = Set("abs", "absolute", "acos", "action", "add", "after", "all", "allocate", "alter", "analyze", "and", "any", "archive", "are", "array", "array_agg", "array_max_cardinality", "as", "asc", "asensitive", "asin", "assertion", "asymmetric", "at", "atan", "atomic", "authorization", "avg", "before", "begin", "begin_frame", "begin_partition", @@ -194,10 +123,23 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "trailing", "transaction", "transactions", "transform", "translate", "translate_regex", "translation", "treat", "trigger", "trim", "trim_array", "true", "truncate", "uescape", "unarchive", "unbounded", "uncache", "under", "undo", "union", "unique", "unknown", "unlock", - "unnest", "unset", "until", "update", "upper", "usage", "use", "user", "value", "value_of", - "values", "var_pop", "var_samp", "varbinary", "varchar", "varying", "versioning", "view", - "when", "whenever", "where", "while", "width_bucket", "window", "with", "within", "without", - "work", "write", "year", "zone") + "unnest", "unset", "until", "update", "upper", "usage", "use", "user", "using", "value", + "value_of", "values", "var_pop", "var_samp", "varbinary", "varchar", "varying", "versioning", + "view", "when", "whenever", "where", "while", "width_bucket", "window", "with", "within", + "without", "work", "write", "year", "zone") + + val reservedKeywordsInAnsiMode = Set("all", "and", "any", "as", "authorization", "both", "case", + "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", + "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", + "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", + "intersect", "into", "is", "join", "leading", "left", "natural", "not", "null", "on", "only", + "or", "order", "outer", "overlaps", "primary", "references", "right", "select", "session_user", + "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "when", + "where", "with") + + val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode + + val nonReservedKeywordsInNonAnsiMode = allCandidateKeywords -- Set("using") test("table identifier") { // Regular names. @@ -220,7 +162,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { test("table identifier - reserved/non-reserved keywords if ANSI mode enabled") { withSQLConf(SQLConf.ANSI_SQL_PARSER.key -> "true") { - reservedKeywordsInAnsi.foreach { keyword => + reservedKeywordsInAnsiMode.foreach { keyword => val errMsg = intercept[ParseException] { parseTableIdentifier(keyword) }.getMessage @@ -229,7 +171,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) } - nonReservedKeywordsInAnsi.foreach { keyword => + nonReservedKeywordsInAnsiMode.foreach { keyword => assert(TableIdentifier(keyword) === parseTableIdentifier(s"$keyword")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.$keyword")) } @@ -246,7 +188,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { } test("table identifier - non reserved keywords") { - nonReservedKeywordsInNonAnsi.foreach { nonReserved => + nonReservedKeywordsInNonAnsiMode.foreach { nonReserved => assert(TableIdentifier(nonReserved) === parseTableIdentifier(nonReserved)) } From d0c84b1a6a639e1e726076a3de5768bfec229775 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Sun, 17 Feb 2019 15:28:10 +0900 Subject: [PATCH 08/18] Update the doc --- docs/sql-reserved-and-non-reserved-key-words.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index 829315b9ab293..eca0b9c5941bc 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -9,7 +9,8 @@ These reserved key words in this ANSI mode basically follow the ANSI SQL-2011 st A full list of reserved/non-reserved key words is as follows: - + + From 7e0a895943bdd749e961c526f4303333af53a319 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 20 Feb 2019 15:55:23 +0900 Subject: [PATCH 09/18] Add ANTI, SEMI, and SETMINUS as non-reserved when ansi=true --- ...sql-reserved-and-non-reserved-key-words.md | 3 ++ .../spark/sql/catalyst/parser/SqlBase.g4 | 28 ++++++++-------- .../parser/TableIdentifierParserSuite.scala | 32 +++++++++---------- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index eca0b9c5941bc..d22ce29bd1aa3 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -22,6 +22,7 @@ A full list of reserved/non-reserved key words is as follows: + @@ -445,6 +446,7 @@ A full list of reserved/non-reserved key words is as follows: + @@ -452,6 +454,7 @@ A full list of reserved/non-reserved key words is as follows: + diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index d9548dc00c5f5..9c1b384b14a4b 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -763,7 +763,7 @@ ansiReserved ; ansiNonReserved - : ADD | AFTER | ALTER | ANALYZE | ARCHIVE | ARRAY | ASC | AT | BETWEEN | BUCKET | BUCKETS | BY | CACHE + : ADD | AFTER | ALTER | ANALYZE | ANTI | ARCHIVE | ARRAY | ASC | AT | BETWEEN | BUCKET | BUCKETS | BY | CACHE | CASCADE | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLECTION | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST | CUBE | CURRENT | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY | DISTRIBUTE @@ -774,8 +774,8 @@ ansiNonReserved | MAP | MSCK | NO | NULLS | OF | OPTION | OPTIONS | OUT | OUTPUTFORMAT | OVER | OVERWRITE | PARTITION | PARTITIONED | PARTITIONS | PERCENT | PERCENTLIT | PIVOT | PRECEDING | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFRESH | RENAME | REPAIR | REPLACE | RESET | RESTRICT - | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SCHEMA | SEPARATED | SERDE - | SERDEPROPERTIES | SET | SETS | SHOW | SKEWED | SORT | SORTED | START | STATISTICS | STORED | STRATIFY + | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SCHEMA | SEMI | SEPARATED | SERDE + | SERDEPROPERTIES | SET | SETMINUS | SETS | SHOW | SKEWED | SORT | SORTED | START | STATISTICS | STORED | STRATIFY | STRUCT | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | TOUCH | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE | UNLOCK | UNSET | USE | VALUES | VIEW | WINDOW ; @@ -784,21 +784,21 @@ nonReserved : ADD | AFTER | ALL | ALTER | ANALYZE | AND | ANY | ARCHIVE | ARRAY | AS | ASC | AT | AUTHORIZATION | BETWEEN | BOTH | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHANGE | CHECK | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLATE | COLLECTION | COLUMN | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS | COMPUTE - | CONCATENATE | CONSTRAINT | COST | CREATE | CUBE | CURRENT | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER - | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES - | DIRECTORY | DISTINCT | DISTRIBUTE | DIV | DROP | ELSE | END | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT - | EXTENDED | EXTERNAL | EXTRACT | FALSE | FETCH | FIELDS | FILEFORMAT | FIRST | FOLLOWING | FOR | FOREIGN | FORMAT - | FORMATTED | FROM | FUNCTION | FUNCTIONS | GLOBAL | GRANT | GROUP | GROUPING | HAVING | IF | IGNORE | IMPORT - | IN | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | INTO | IS | ITEMS | KEYS | LAST | LATERAL - | LAZY | LEADING | LIKE | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO + | CONCATENATE | CONSTRAINT | COST | CREATE | CUBE | CURRENT | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP + | CURRENT_USER | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS + | DIRECTORIES | DIRECTORY | DISTINCT | DISTRIBUTE | DIV | DROP | ELSE | END | ESCAPED | EXCHANGE | EXISTS | EXPLAIN + | EXPORT | EXTENDED | EXTERNAL | EXTRACT | FALSE | FETCH | FIELDS | FILEFORMAT | FIRST | FOLLOWING | FOR | FOREIGN + | FORMAT | FORMATTED | FROM | FUNCTION | FUNCTIONS | GLOBAL | GRANT | GROUP | GROUPING | HAVING | IF | IGNORE + | IMPORT | IN | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | INTO | IS | ITEMS | KEYS | LAST + | LATERAL | LAZY | LEADING | LIKE | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO | MAP | MSCK | NO | NOT | NULL | NULLS | OF | ONLY | OPTION | OPTIONS | OR | ORDER | OUT | OUTER | OUTPUTFORMAT | OVER | OVERLAPS | OVERWRITE | PARTITION | PARTITIONED | PARTITIONS | PERCENTLIT | PIVOT | POSITION | PRECEDING | PRIMARY | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFERENCES | REFRESH | RENAME | REPAIR | REPLACE | RESET | RESTRICT | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS - | SELECT | SEPARATED | SERDE | SERDEPROPERTIES | SESSION_USER | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED | START - | STATISTICS | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED - | THEN | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED - | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH + | SELECT | SEPARATED | SERDE | SERDEPROPERTIES | SESSION_USER | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED + | START | STATISTICS | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY + | TERMINATED | THEN | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE + | UNBOUNDED | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; SELECT: 'SELECT'; diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index 289657d2f08dd..527a845b5831b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -61,7 +61,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { // All the keywords in `docs/sql-reserved-and-non-reserved-key-words.md` are listed below: val allCandidateKeywords = Set("abs", "absolute", "acos", "action", "add", "after", - "all", "allocate", "alter", "analyze", "and", "any", "archive", "are", "array", "array_agg", + "all", "allocate", "alter", "analyze", "and", "anti", "any", "archive", "are", "array", "array_agg", "array_max_cardinality", "as", "asc", "asensitive", "asin", "assertion", "asymmetric", "at", "atan", "atomic", "authorization", "avg", "before", "begin", "begin_frame", "begin_partition", "between", "bigint", "binary", "bit", "bit_length", "blob", "boolean", "both", "breadth", @@ -112,21 +112,21 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "repeat", "replace", "reset", "resignal", "restrict", "result", "return", "returns", "revoke", "right", "rlike", "role", "roles", "rollback", "rollup", "routine", "row", "row_number", "rows", "running", "savepoint", "schema", "scope", "scroll", "search", "second", "section", "seek", - "select", "sensitive", "separated", "serde", "serdeproperties", "session", "session_user", - "set", "sets", "show", "signal", "similar", "sin", "sinh", "size", "skewed", "skip", "smallint", - "some", "sort", "sorted", "space", "specific", "specifictype", "sql", "sqlcode", "sqlerror", - "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", "state", "static", "statistics", - "stddev_pop", "stddev_samp", "stored", "stratify", "struct", "submultiset", "subset", - "substring", "substring_regex", "succeeds", "sum", "symmetric", "system", "system_time", - "system_user", "table", "tables", "tablesample", "tan", "tanh", "tblproperties", "temporary", - "terminated", "then", "time", "timestamp", "timezone_hour", "timezone_minute", "to", "touch", - "trailing", "transaction", "transactions", "transform", "translate", "translate_regex", - "translation", "treat", "trigger", "trim", "trim_array", "true", "truncate", "uescape", - "unarchive", "unbounded", "uncache", "under", "undo", "union", "unique", "unknown", "unlock", - "unnest", "unset", "until", "update", "upper", "usage", "use", "user", "using", "value", - "value_of", "values", "var_pop", "var_samp", "varbinary", "varchar", "varying", "versioning", - "view", "when", "whenever", "where", "while", "width_bucket", "window", "with", "within", - "without", "work", "write", "year", "zone") + "select", "semi", "sensitive", "separated", "serde", "serdeproperties", "session", + "session_user", "set", "setminus", "sets", "show", "signal", "similar", "sin", "sinh", "size", + "skewed", "skip", "smallint", "some", "sort", "sorted", "space", "specific", "specifictype", + "sql", "sqlcode", "sqlerror", "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", + "state", "static", "statistics", "stddev_pop", "stddev_samp", "stored", "stratify", "struct", + "submultiset", "subset", "substring", "substring_regex", "succeeds", "sum", "symmetric", + "system", "system_time", "system_user", "table", "tables", "tablesample", "tan", "tanh", + "tblproperties", "temporary", "terminated", "then", "time", "timestamp", "timezone_hour", + "timezone_minute", "to", "touch", "trailing", "transaction", "transactions", "transform", + "translate", "translate_regex", "translation", "treat", "trigger", "trim", "trim_array", "true", + "truncate", "uescape", "unarchive", "unbounded", "uncache", "under", "undo", "union", "unique", + "unknown", "unlock", "unnest", "unset", "until", "update", "upper", "usage", "use", "user", + "using", "value", "value_of", "values", "var_pop", "var_samp", "varbinary", "varchar", + "varying", "versioning", "view", "when", "whenever", "where", "while", "width_bucket", "window", + "with", "within", "without", "work", "write", "year", "zone") val reservedKeywordsInAnsiMode = Set("all", "and", "any", "as", "authorization", "both", "case", "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", From 30f701319da6c54a9389d6613bcede2c7b3187e7 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 20 Feb 2019 16:44:41 +0900 Subject: [PATCH 10/18] Add comments --- .../spark/sql/catalyst/parser/SqlBase.g4 | 11 ++- .../parser/TableIdentifierParserSuite.scala | 77 ++++++++++--------- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 9c1b384b14a4b..36fb870c0f30a 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -751,9 +751,13 @@ number | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral ; +// A list of reserved keywords in Spark SQL. These keywords are reserved when `spark.sql.parser.ansi.enabled` = true. +// Currently, we only reserve the ANSI keywords below that almost all the ANSI SQL standards (SQL-92, SQL-99, +// SQL-2003, SQL-2008, SQL-2011, and SQL-2016) and PostgreSQL reserve. +// +// NOTE: The ANTLR tokens in `SqlBase.g4` must exist in either `ansiReserved` or `ansiNonReserved`. Therefore, +// when one adds a new token in this file, ones must update one of the two rules, too. ansiReserved - // A list of reserved keywords in SparkSQL. These keywords are reserved in all the ANSI SQL standards - // (SQL-92, SQL-99, SQL-2003, SQL-2008, SQL-2011, and SQL-2016) and PostgreSQL. : ALL | AND | ANY | AS | AUTHORIZATION | BOTH | CASE | CAST | CHECK | COLLATE | COLUMN | CONSTRAINT | CREATE | CROSS | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DISTINCT | ELSE | END | EXCEPT | FALSE | FETCH | FOR | FOREIGN | FROM | FULL | GRANT | GROUP | HAVING | IN | INNER | INTERSECT | INTO | IS @@ -762,6 +766,9 @@ ansiReserved | USING | WHEN | WHERE | WITH ; +// When `spark.sql.parser.ansi.enabled` = true, the `ansiNonReserved` keywords can be used for identifiers. +// Otherwise (`spark.sql.parser.ansi.enabled` = false), we follow the existing Spark SQL behaviour until v3.0: +// the `nonReserved` keywords can be used instead. ansiNonReserved : ADD | AFTER | ALTER | ANALYZE | ANTI | ARCHIVE | ARRAY | ASC | AT | BETWEEN | BUCKET | BUCKETS | BY | CACHE | CASCADE | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLECTION | COLUMNS | COMMENT | COMMIT diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index 527a845b5831b..ea1ae0a0c8564 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -61,43 +61,44 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { // All the keywords in `docs/sql-reserved-and-non-reserved-key-words.md` are listed below: val allCandidateKeywords = Set("abs", "absolute", "acos", "action", "add", "after", - "all", "allocate", "alter", "analyze", "and", "anti", "any", "archive", "are", "array", "array_agg", - "array_max_cardinality", "as", "asc", "asensitive", "asin", "assertion", "asymmetric", "at", - "atan", "atomic", "authorization", "avg", "before", "begin", "begin_frame", "begin_partition", - "between", "bigint", "binary", "bit", "bit_length", "blob", "boolean", "both", "breadth", - "bucket", "buckets", "by", "cache", "call", "called", "cardinality", "cascade", "cascaded", - "case", "cast", "catalog", "ceil", "ceiling", "change", "char", "char_length", "character", - "character_length", "check", "classifier", "clear", "clob", "close", "cluster", "clustered", - "coalesce", "codegen", "collate", "collation", "collect", "collection", "column", "columns", - "comment", "commit", "compact", "compactions", "compute", "concatenate", "condition", "connect", - "connection", "constraint", "constraints", "constructor", "contains", "continue", "convert", - "copy", "corr", "corresponding", "cos", "cosh", "cost", "count", "covar_pop", "covar_samp", - "create", "cross", "cube", "cume_dist", "current", "current_catalog", "current_date", - "current_default_transform_group", "current_path", "current_role", "current_row", - "current_schema", "current_time", "current_timestamp", "current_transform_group_for_type", - "current_user", "cursor", "cycle", "data", "database", "databases", "date", "day", - "dbproperties", "deallocate", "dec", "decfloat", "decimal", "declare", "default", "deferrable", - "deferred", "define", "defined", "delete", "delimited", "dense_rank", "depth", "deref", "desc", - "describe", "descriptor", "deterministic", "dfs", "diagnostics", "directories", "directory", - "disconnect", "distinct", "distribute", "div", "do", "domain", "double", "drop", "dynamic", - "each", "element", "else", "elseif", "empty", "end", "end_frame", "end_partition", "equals", - "escape", "escaped", "every", "except", "exception", "exchange", "exec", "execute", "exists", - "exit", "exp", "explain", "export", "extended", "external", "extract", "false", "fetch", - "fields", "fileformat", "filter", "first", "first_value", "float", "following", "for", - "foreign", "format", "formatted", "found", "frame_row", "free", "from", "full", "function", - "functions", "fusion", "general", "get", "global", "go", "goto", "grant", "group", "grouping", - "groups", "handler", "having", "hold", "hour", "identity", "if", "ignore", "immediate", - "import", "in", "index", "indexes", "indicator", "initial", "initially", "inner", "inout", - "inpath", "input", "inputformat", "insensitive", "insert", "int", "integer", "intersect", - "intersection", "interval", "into", "is", "isolation", "items", "iterate", "join", "json_array", - "json_arrayagg", "json_exists", "json_object", "json_objectagg", "json_query", "json_table", - "json_table_primitive", "json_value", "key", "keys", "lag", "language", "large", "last", - "last_value", "lateral", "lazy", "lead", "leading", "leave", "left", "level", "like", - "like_regex", "limit", "lines", "list", "listagg", "ln", "load", "local", "localtime", - "localtimestamp", "location", "locator", "lock", "locks", "log", "log10", "logical", "loop", - "lower", "macro", "map", "match", "match_number", "match_recognize", "matches", "max", "member", - "merge", "method", "min", "minute", "mod", "modifies", "module", "month", "msck", "multiset", - "names", "national", "natural", "nchar", "nclob", "new", "next", "no", "none", "normalize", + "all", "allocate", "alter", "analyze", "and", "anti", "any", "archive", "are", "array", + "array_agg", "array_max_cardinality", "as", "asc", "asensitive", "asin", "assertion", + "asymmetric", "at", "atan", "atomic", "authorization", "avg", "before", "begin", "begin_frame", + "begin_partition", "between", "bigint", "binary", "bit", "bit_length", "blob", "boolean", + "both", "breadth", "bucket", "buckets", "by", "cache", "call", "called", "cardinality", + "cascade", "cascaded", "case", "cast", "catalog", "ceil", "ceiling", "change", "char", + "char_length", "character", "character_length", "check", "classifier", "clear", "clob", "close", + "cluster", "clustered", "coalesce", "codegen", "collate", "collation", "collect", "collection", + "column", "columns", "comment", "commit", "compact", "compactions", "compute", "concatenate", + "condition", "connect", "connection", "constraint", "constraints", "constructor", "contains", + "continue", "convert", "copy", "corr", "corresponding", "cos", "cosh", "cost", "count", + "covar_pop", "covar_samp", "create", "cross", "cube", "cume_dist", "current", "current_catalog", + "current_date", "current_default_transform_group", "current_path", "current_role", + "current_row", "current_schema", "current_time", "current_timestamp", + "current_transform_group_for_type", "current_user", "cursor", "cycle", "data", "database", + "databases", "date", "day", "dbproperties", "deallocate", "dec", "decfloat", "decimal", + "declare", "default", "deferrable", "deferred", "define", "defined", "delete", "delimited", + "dense_rank", "depth", "deref", "desc", "describe", "descriptor", "deterministic", "dfs", + "diagnostics", "directories", "directory", "disconnect", "distinct", "distribute", "div", "do", + "domain", "double", "drop", "dynamic", "each", "element", "else", "elseif", "empty", "end", + "end_frame", "end_partition", "equals", "escape", "escaped", "every", "except", "exception", + "exchange", "exec", "execute", "exists", "exit", "exp", "explain", "export", "extended", + "external", "extract", "false", "fetch", "fields", "fileformat", "filter", "first", + "first_value", "float", "following", "for", "foreign", "format", "formatted", "found", + "frame_row", "free", "from", "full", "function", "functions", "fusion", "general", "get", + "global", "go", "goto", "grant", "group", "grouping", "groups", "handler", "having", "hold", + "hour", "identity", "if", "ignore", "immediate", "import", "in", "index", "indexes", + "indicator", "initial", "initially", "inner", "inout", "inpath", "input", "inputformat", + "insensitive", "insert", "int", "integer", "intersect", "intersection", "interval", "into", + "is", "isolation", "items", "iterate", "join", "json_array", "json_arrayagg", "json_exists", + "json_object", "json_objectagg", "json_query", "json_table", "json_table_primitive", + "json_value", "key", "keys", "lag", "language", "large", "last", "last_value", "lateral", + "lazy", "lead", "leading", "leave", "left", "level", "like", "like_regex", "limit", "lines", + "list", "listagg", "ln", "load", "local", "localtime", "localtimestamp", "location", + "locator", "lock", "locks", "log", "log10", "logical", "loop", "lower", "macro", "map", + "match", "match_number", "match_recognize", "matches", "max", "member", "merge", "method", + "min", "minute", "mod", "modifies", "module", "month", "msck", "multiset", "names", + "national", "natural", "nchar", "nclob", "new", "next", "no", "none", "normalize", "not", "nth_value", "ntile", "null", "nullif", "nulls", "numeric", "object", "occurrences_regex", "octet_length", "of", "offset", "old", "omit", "on", "one", "only", "open", "option", "options", "or", "order", "ordinality", "out", "outer", "output", "outputformat", @@ -113,7 +114,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "right", "rlike", "role", "roles", "rollback", "rollup", "routine", "row", "row_number", "rows", "running", "savepoint", "schema", "scope", "scroll", "search", "second", "section", "seek", "select", "semi", "sensitive", "separated", "serde", "serdeproperties", "session", - "session_user", "set", "setminus", "sets", "show", "signal", "similar", "sin", "sinh", "size", + "session_user", "set", "setminus", "sets", "show", "signal", "similar", "sin", "sinh", "size", "skewed", "skip", "smallint", "some", "sort", "sorted", "space", "specific", "specifictype", "sql", "sqlcode", "sqlerror", "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", "state", "static", "statistics", "stddev_pop", "stddev_samp", "stored", "stratify", "struct", From 8c3e85987b6ae7ba86f7dc7ebdb8e9d9714d835f Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 20 Feb 2019 20:27:05 +0900 Subject: [PATCH 11/18] Add strictNonReserved --- .../org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 36fb870c0f30a..91e9915821d8a 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -726,8 +726,7 @@ qualifiedName identifier : {ansi}? ansiReserved | strictIdentifier - | ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON - | UNION | INTERSECT | EXCEPT | SETMINUS + | {!ansi}? strictNonReserved ; strictIdentifier @@ -808,6 +807,11 @@ nonReserved | UNBOUNDED | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; +strictNonReserved + : ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON + | UNION | INTERSECT | EXCEPT | SETMINUS + ; + SELECT: 'SELECT'; FROM: 'FROM'; ADD: 'ADD'; From 4c6b9f65fea99d8cdd2fd61d7fd6eae1300d752c Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Thu, 21 Feb 2019 15:20:52 +0900 Subject: [PATCH 12/18] Fix --- ...sql-reserved-and-non-reserved-key-words.md | 26 +++---- .../spark/sql/catalyst/parser/SqlBase.g4 | 69 ++++++++++--------- .../sql/catalyst/parser/AstBuilder.scala | 68 ++++++++++-------- .../parser/TableIdentifierParserSuite.scala | 18 +++-- .../spark/sql/execution/SparkSqlParser.scala | 33 ++++----- 5 files changed, 115 insertions(+), 99 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index d22ce29bd1aa3..7efd77434a6f2 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -112,7 +112,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -179,7 +179,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -209,7 +209,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -237,7 +237,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -246,7 +246,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -254,7 +254,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -276,7 +276,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -309,6 +309,7 @@ A full list of reserved/non-reserved key words is as follows: + @@ -318,7 +319,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -340,7 +341,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -426,7 +427,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -454,7 +455,6 @@ A full list of reserved/non-reserved key words is as follows: - @@ -531,7 +531,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -543,7 +543,7 @@ A full list of reserved/non-reserved key words is as follows: - + diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 91e9915821d8a..aec1911df0e0d 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -724,14 +724,17 @@ qualifiedName ; identifier - : {ansi}? ansiReserved - | strictIdentifier - | {!ansi}? strictNonReserved + : strictIdentifier + // We can't move the tokens below into `strictIdentifier` because we need to parse some SQL texts, + // e.g., SELECT * FROM t1 CROSS JOIN t2 + | ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON + | UNION | INTERSECT | EXCEPT | SETMINUS ; strictIdentifier : IDENTIFIER #unquotedIdentifier | quotedIdentifier #quotedIdentifierAlternative + | {ansi}? ansiReserved #unquotedIdentifier | {ansi}? ansiNonReserved #unquotedIdentifier | {!ansi}? nonReserved #unquotedIdentifier ; @@ -750,18 +753,24 @@ number | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral ; -// A list of reserved keywords in Spark SQL. These keywords are reserved when `spark.sql.parser.ansi.enabled` = true. -// Currently, we only reserve the ANSI keywords below that almost all the ANSI SQL standards (SQL-92, SQL-99, -// SQL-2003, SQL-2008, SQL-2011, and SQL-2016) and PostgreSQL reserve. +// NOTE: You must follow a rule below when you add a new ANTLR taken in this file: +// - All the ANTLR tokens except for some keywords (ANTI, FULL, INNER, LEFT, SEMI, RIGHT, NATURAL, JOIN, CROSS, ON, +// UNION, INTERSECT, EXCEPT, and SETMINUS) +// = UNION(`ansiReserved`, `ansiNonReserved`) +// = `nonReserved` // -// NOTE: The ANTLR tokens in `SqlBase.g4` must exist in either `ansiReserved` or `ansiNonReserved`. Therefore, -// when one adds a new token in this file, ones must update one of the two rules, too. +// Let's say you add a new token `NEWTOKEN` and this is not reserved regardless of a `spark.sql.parser.ansi.enabled` +// value. In this case, you must add a token `NEWTOKEN` in both `ansiNonReserved` and `nonReserved`. + +// A list of the reserved keywords below in Spark SQL. These keywords cannot be used for identifiers +// when `spark.sql.parser.ansi.enabled` = true. Currently, we only reserve the ANSI keywords +// that almost all the ANSI SQL standards (SQL-92, SQL-99, SQL-2003, SQL-2008, SQL-2011, and SQL-2016) +// and PostgreSQL reserve. ansiReserved : ALL | AND | ANY | AS | AUTHORIZATION | BOTH | CASE | CAST | CHECK | COLLATE | COLUMN | CONSTRAINT | CREATE - | CROSS | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DISTINCT | ELSE | END | EXCEPT - | FALSE | FETCH | FOR | FOREIGN | FROM | FULL | GRANT | GROUP | HAVING | IN | INNER | INTERSECT | INTO | IS - | JOIN | LEADING | LEFT | NATURAL | NOT | NULL | ON | ONLY | OR | ORDER | OUTER | OVERLAPS | PRIMARY - | REFERENCES | RIGHT | SELECT | SESSION_USER | SOME | TABLE | THEN | TO | TRAILING | UNION | UNIQUE | USER + | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DISTINCT | ELSE | END | FALSE | FETCH | FOR + | FOREIGN | FROM | GRANT | GROUP | HAVING | IN | INTO | IS | LEADING | NOT | NULL | ONLY | OR | ORDER | OUTER + | OVERLAPS | PRIMARY | REFERENCES | SELECT | SESSION_USER | SOME | TABLE | THEN | TO | TRAILING | UNIQUE | USER | USING | WHEN | WHERE | WITH ; @@ -769,21 +778,20 @@ ansiReserved // Otherwise (`spark.sql.parser.ansi.enabled` = false), we follow the existing Spark SQL behaviour until v3.0: // the `nonReserved` keywords can be used instead. ansiNonReserved - : ADD | AFTER | ALTER | ANALYZE | ANTI | ARCHIVE | ARRAY | ASC | AT | BETWEEN | BUCKET | BUCKETS | BY | CACHE - | CASCADE | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLECTION | COLUMNS | COMMENT | COMMIT - | COMPACT | COMPACTIONS | COMPUTE | CONCATENATE | COST | CUBE | CURRENT | DATA | DATABASE | DATABASES - | DBPROPERTIES | DEFINED | DELETE | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY | DISTRIBUTE - | DIV | DROP | ESCAPED | EXCHANGE | EXISTS | EXPLAIN | EXPORT | EXTENDED | EXTERNAL | EXTRACT | FIELDS - | FILEFORMAT | FIRST | FOLLOWING | FORMAT | FORMATTED | FUNCTION | FUNCTIONS | GLOBAL | GROUPING | IF - | IGNORE | IMPORT | INDEX | INDEXES | INPATH | INPUTFORMAT | INSERT | INTERVAL | ITEMS | KEYS | LAST - | LATERAL | LAZY | LIKE | LIMIT | LINES | LIST | LOAD | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO - | MAP | MSCK | NO | NULLS | OF | OPTION | OPTIONS | OUT | OUTPUTFORMAT | OVER | OVERWRITE | PARTITION - | PARTITIONED | PARTITIONS | PERCENT | PERCENTLIT | PIVOT | PRECEDING | PRINCIPALS | PURGE | RANGE - | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFRESH | RENAME | REPAIR | REPLACE | RESET | RESTRICT - | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SCHEMA | SEMI | SEPARATED | SERDE - | SERDEPROPERTIES | SET | SETMINUS | SETS | SHOW | SKEWED | SORT | SORTED | START | STATISTICS | STORED | STRATIFY - | STRUCT | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | TOUCH | TRANSACTION | TRANSACTIONS - | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE | UNLOCK | UNSET | USE | VALUES | VIEW | WINDOW + : ADD | AFTER | ALTER | ANALYZE | ARCHIVE | ARRAY | ASC | AT | BETWEEN | BUCKET | BUCKETS | BY | CACHE | CASCADE + | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLECTION | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS + | COMPUTE | CONCATENATE | COST | CUBE | CURRENT | DATA | DATABASE | DATABASES | DBPROPERTIES | DEFINED | DELETE + | DELIMITED | DESC | DESCRIBE | DFS | DIRECTORIES | DIRECTORY | DISTRIBUTE | DIV | DROP | ESCAPED | EXCHANGE + | EXISTS | EXPLAIN | EXPORT | EXTENDED | EXTERNAL | EXTRACT | FIELDS | FILEFORMAT | FIRST | FOLLOWING | FORMAT + | FORMATTED | FUNCTION | FUNCTIONS | GLOBAL | GROUPING | IF | IGNORE | IMPORT | INDEX | INDEXES | INPATH + | INPUTFORMAT | INSERT | INTERVAL | ITEMS | KEYS | LAST | LATERAL | LAZY | LIKE | LIMIT | LINES | LIST | LOAD + | LOCAL | LOCATION | LOCK | LOCKS | LOGICAL | MACRO | MAP | MSCK | NO | NULLS | OF | OPTION | OPTIONS | OUT + | OUTPUTFORMAT | OVER | OVERWRITE | PARTITION | PARTITIONED | PARTITIONS | PERCENT | PERCENTLIT | PIVOT | PRECEDING + | PRINCIPALS | PURGE | RANGE | RECORDREADER | RECORDWRITER | RECOVER | REDUCE | REFRESH | RENAME | REPAIR | REPLACE + | RESET | RESTRICT | REVOKE | RLIKE | ROLE | ROLES | ROLLBACK | ROLLUP | ROW | ROWS | SCHEMA | SEPARATED | SERDE + | SERDEPROPERTIES | SET | SETS | SHOW | SKEWED | SORT | SORTED | START | STATISTICS | STORED | STRATIFY | STRUCT + | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | TOUCH | TRANSACTION | TRANSACTIONS | TRANSFORM + | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE | UNLOCK | UNSET | USE | VALUES | VIEW | WINDOW ; nonReserved @@ -804,12 +812,7 @@ nonReserved | SELECT | SEPARATED | SERDE | SERDEPROPERTIES | SESSION_USER | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED | START | STATISTICS | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | THEN | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE - | UNBOUNDED | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH - ; - -strictNonReserved - : ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON - | UNION | INTERSECT | EXCEPT | SETMINUS + | UNBOUNDED | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | USING | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; SELECT: 'SELECT'; diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 853068a6cdf58..c9d46644d3f26 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -51,8 +51,6 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging def this() = this(new SQLConf()) - def ansi: Boolean = conf.ansiParserEnabled - protected def typedVisit[T](ctx: ParseTree): T = { ctx.accept(this).asInstanceOf[T] } @@ -273,7 +271,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitPartitionSpec( ctx: PartitionSpecContext): Map[String, Option[String]] = withOrigin(ctx) { val parts = ctx.partitionVal.asScala.map { pVal => - val name = pVal.identifier.getText + val name = visitIdentifier(pVal.identifier) val value = Option(pVal.constant).map(visitStringConstant) name -> value } @@ -572,7 +570,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging // Collect all window specifications defined in the WINDOW clause. val baseWindowMap = ctx.namedWindow.asScala.map { wCtx => - (wCtx.identifier.getText, typedVisit[WindowSpec](wCtx.windowSpec)) + (visitIdentifier(wCtx.identifier), typedVisit[WindowSpec](wCtx.windowSpec)) }.toMap // Handle cases like @@ -647,11 +645,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging .flatMap(_.namedExpression.asScala) .map(typedVisit[Expression]) val pivotColumn = if (ctx.pivotColumn.identifiers.size == 1) { - UnresolvedAttribute.quoted(ctx.pivotColumn.identifier.getText) + UnresolvedAttribute.quoted(visitIdentifier(ctx.pivotColumn.identifier)) } else { CreateStruct( ctx.pivotColumn.identifiers.asScala.map( - identifier => UnresolvedAttribute.quoted(identifier.getText))) + identifier => UnresolvedAttribute.quoted(visitIdentifier(identifier)))) } val pivotValues = ctx.pivotValues.asScala.map(visitPivotValue) Pivot(None, pivotColumn, pivotValues, aggregates, query) @@ -663,7 +661,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitPivotValue(ctx: PivotValueContext): Expression = withOrigin(ctx) { val e = expression(ctx.expression) if (ctx.identifier != null) { - Alias(e, ctx.identifier.getText)() + Alias(e, visitIdentifier(ctx.identifier))() } else { e } @@ -834,8 +832,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } val tvf = UnresolvedTableValuedFunction( - func.identifier.getText, func.expression.asScala.map(expression), aliases) - tvf.optionalMap(func.tableAlias.strictIdentifier)(aliasPlan) + visitIdentifier(func.identifier), func.expression.asScala.map(expression), aliases) + tvf.optionalMap(strictIdentifier(func.tableAlias.strictIdentifier))(aliasPlan) } /** @@ -860,7 +858,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } val table = UnresolvedInlineTable(aliases, rows) - table.optionalMap(ctx.tableAlias.strictIdentifier)(aliasPlan) + table.optionalMap(strictIdentifier(ctx.tableAlias.strictIdentifier))(aliasPlan) } /** @@ -900,8 +898,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging /** * Create an alias ([[SubqueryAlias]]) for a [[LogicalPlan]]. */ - private def aliasPlan(alias: ParserRuleContext, plan: LogicalPlan): LogicalPlan = { - SubqueryAlias(alias.getText, plan) + private def aliasPlan(alias: String, plan: LogicalPlan): LogicalPlan = { + SubqueryAlias(alias, plan) } /** @@ -910,7 +908,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ private def mayApplyAliasPlan(tableAlias: TableAliasContext, plan: LogicalPlan): LogicalPlan = { if (tableAlias.strictIdentifier != null) { - val subquery = SubqueryAlias(tableAlias.strictIdentifier.getText, plan) + val subquery = SubqueryAlias(strictIdentifier(tableAlias.strictIdentifier), plan) if (tableAlias.identifierList != null) { val columnNames = visitIdentifierList(tableAlias.identifierList) UnresolvedSubqueryColumnAliases(columnNames, subquery) @@ -933,7 +931,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a Sequence of Strings for an identifier list. */ override def visitIdentifierSeq(ctx: IdentifierSeqContext): Seq[String] = withOrigin(ctx) { - ctx.identifier.asScala.map(_.getText) + ctx.identifier.asScala.map(visitIdentifier) } /* ******************************************************************************************** @@ -976,7 +974,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Both un-targeted (global) and targeted aliases are supported. */ override def visitStar(ctx: StarContext): Expression = withOrigin(ctx) { - UnresolvedStar(Option(ctx.qualifiedName()).map(_.identifier.asScala.map(_.getText))) + UnresolvedStar(Option(ctx.qualifiedName()).map(_.identifier.asScala.map(visitIdentifier))) } /** @@ -986,7 +984,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitNamedExpression(ctx: NamedExpressionContext): Expression = withOrigin(ctx) { val e = expression(ctx.expression) if (ctx.identifier != null) { - Alias(e, ctx.identifier.getText)() + Alias(e, visitIdentifier(ctx.identifier))() } else if (ctx.identifierList != null) { MultiAlias(e, visitIdentifierList(ctx.identifierList)) } else { @@ -1328,7 +1326,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a function database (optional) and name pair. */ protected def visitFunctionName(ctx: QualifiedNameContext): FunctionIdentifier = { - ctx.identifier().asScala.map(_.getText) match { + ctx.identifier().asScala.map(visitIdentifier) match { case Seq(db, fn) => FunctionIdentifier(fn, Option(db)) case Seq(fn) => FunctionIdentifier(fn, None) case other => throw new ParseException(s"Unsupported function name '${ctx.getText}'", ctx) @@ -1352,7 +1350,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a reference to a window frame, i.e. [[WindowSpecReference]]. */ override def visitWindowRef(ctx: WindowRefContext): WindowSpecReference = withOrigin(ctx) { - WindowSpecReference(ctx.identifier.getText) + WindowSpecReference(visitIdentifier(ctx.identifier)) } /** @@ -1406,13 +1404,29 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } } + private def strictIdentifier(ctx: StrictIdentifierContext): String = { + if (ctx != null) { + val keyword = ctx.getText + ctx match { + case u: UnquotedIdentifierContext => + if (u.ansiReserved() != null) { + throw new ParseException( + s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) + } + case _ => + } + keyword + } else { + null + } + } + override def visitIdentifier(ctx: IdentifierContext): String = withOrigin(ctx) { - val keyword = ctx.getText - if (ansi && ctx.ansiReserved() != null) { - throw new ParseException( - s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) + if (ctx.strictIdentifier() != null) { + strictIdentifier(ctx.strictIdentifier()) + } else { + ctx.getText } - keyword } /** @@ -1562,7 +1576,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ override def visitTypeConstructor(ctx: TypeConstructorContext): Literal = withOrigin(ctx) { val value = string(ctx.STRING) - val valueType = ctx.identifier.getText.toUpperCase(Locale.ROOT) + val valueType = visitIdentifier(ctx.identifier).toUpperCase(Locale.ROOT) def toLiteral[T](f: UTF8String => Option[T], t: DataType): Literal = { f(UTF8String.fromString(value)).map(Literal(_, t)).getOrElse { throw new ParseException(s"Cannot parse the $valueType value: $value", ctx) @@ -1766,7 +1780,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Resolve/create a primitive type. */ override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType = withOrigin(ctx) { - val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT) + val dataType = visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) (dataType, ctx.INTEGER_VALUE().asScala.toList) match { case ("boolean", Nil) => BooleanType case ("tinyint" | "byte", Nil) => ByteType @@ -1838,7 +1852,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } StructField( - identifier.getText, + visitIdentifier(identifier), cleanedDataType, nullable = true, builder.build()) @@ -1864,7 +1878,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ override def visitComplexColType(ctx: ComplexColTypeContext): StructField = withOrigin(ctx) { import ctx._ - val structField = StructField(identifier.getText, typedVisit(dataType), nullable = true) + val structField = StructField(visitIdentifier(identifier), typedVisit(dataType), nullable = true) if (STRING == null) structField else structField.withComment(string(STRING)) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index ea1ae0a0c8564..ebfc2fecf2f27 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -97,7 +97,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "list", "listagg", "ln", "load", "local", "localtime", "localtimestamp", "location", "locator", "lock", "locks", "log", "log10", "logical", "loop", "lower", "macro", "map", "match", "match_number", "match_recognize", "matches", "max", "member", "merge", "method", - "min", "minute", "mod", "modifies", "module", "month", "msck", "multiset", "names", + "min", "minus", "minute", "mod", "modifies", "module", "month", "msck", "multiset", "names", "national", "natural", "nchar", "nclob", "new", "next", "no", "none", "normalize", "not", "nth_value", "ntile", "null", "nullif", "nulls", "numeric", "object", "occurrences_regex", "octet_length", "of", "offset", "old", "omit", "on", "one", "only", "open", @@ -114,7 +114,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "right", "rlike", "role", "roles", "rollback", "rollup", "routine", "row", "row_number", "rows", "running", "savepoint", "schema", "scope", "scroll", "search", "second", "section", "seek", "select", "semi", "sensitive", "separated", "serde", "serdeproperties", "session", - "session_user", "set", "setminus", "sets", "show", "signal", "similar", "sin", "sinh", "size", + "session_user", "set", "sets", "show", "signal", "similar", "sin", "sinh", "size", "skewed", "skip", "smallint", "some", "sort", "sorted", "space", "specific", "specifictype", "sql", "sqlcode", "sqlerror", "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", "state", "static", "statistics", "stddev_pop", "stddev_samp", "stored", "stratify", "struct", @@ -130,17 +130,15 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "with", "within", "without", "work", "write", "year", "zone") val reservedKeywordsInAnsiMode = Set("all", "and", "any", "as", "authorization", "both", "case", - "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", - "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", - "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", - "intersect", "into", "is", "join", "leading", "left", "natural", "not", "null", "on", "only", - "or", "order", "outer", "overlaps", "primary", "references", "right", "select", "session_user", - "some", "table", "then", "to", "trailing", "union", "unique", "user", "using", "when", - "where", "with") + "cast", "check", "collate", "column", "constraint", "create", "current_date", "current_time", + "current_timestamp", "current_user", "distinct", "else", "end", "false", "fetch", "for", + "foreign", "from", "grant", "group", "having", "in", "into", "is", "leading", "not", "null", + "only", "or", "order", "outer", "overlaps", "primary", "references", "select", "session_user", + "some", "table", "then", "to", "trailing", "unique", "user", "using", "when", "where", "with") val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode - val nonReservedKeywordsInNonAnsiMode = allCandidateKeywords -- Set("using") + val nonReservedKeywordsInNonAnsiMode = allCandidateKeywords test("table identifier") { // Regular names. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 8deb55b00a9d3..69f1701d915fc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -116,8 +116,9 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { } } if (ctx.identifier != null && - ctx.identifier.getText.toLowerCase(Locale.ROOT) != "noscan") { - throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`", ctx) + visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) != "noscan") { + throw new ParseException("Expected `NOSCAN` instead of " + + s"`${visitIdentifier(ctx.identifier)}`", ctx) } val table = visitTableIdentifier(ctx.tableIdentifier) @@ -637,7 +638,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitCreateDatabase(ctx: CreateDatabaseContext): LogicalPlan = withOrigin(ctx) { CreateDatabaseCommand( - ctx.identifier.getText, + visitIdentifier(ctx.identifier), ctx.EXISTS != null, Option(ctx.locationSpec).map(visitLocationSpec), Option(ctx.comment).map(string), @@ -655,7 +656,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { override def visitSetDatabaseProperties( ctx: SetDatabasePropertiesContext): LogicalPlan = withOrigin(ctx) { AlterDatabasePropertiesCommand( - ctx.identifier.getText, + visitIdentifier(ctx.identifier), visitPropertyKeyValues(ctx.tablePropertyList)) } @@ -668,7 +669,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { * }}} */ override def visitDropDatabase(ctx: DropDatabaseContext): LogicalPlan = withOrigin(ctx) { - DropDatabaseCommand(ctx.identifier.getText, ctx.EXISTS != null, ctx.CASCADE != null) + DropDatabaseCommand(visitIdentifier(ctx.identifier), ctx.EXISTS != null, ctx.CASCADE != null) } /** @@ -680,7 +681,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { * }}} */ override def visitDescribeDatabase(ctx: DescribeDatabaseContext): LogicalPlan = withOrigin(ctx) { - DescribeDatabaseCommand(ctx.identifier.getText, ctx.EXTENDED != null) + DescribeDatabaseCommand(visitIdentifier(ctx.identifier), ctx.EXTENDED != null) } /** @@ -704,7 +705,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitShowFunctions(ctx: ShowFunctionsContext): LogicalPlan = withOrigin(ctx) { import ctx._ - val (user, system) = Option(ctx.identifier).map(_.getText.toLowerCase(Locale.ROOT)) match { + val (user, system) = Option(ctx.identifier).map(i => visitIdentifier(i).toLowerCase(Locale.ROOT)) match { case None | Some("all") => (true, true) case Some("system") => (false, true) case Some("user") => (true, false) @@ -734,7 +735,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitCreateFunction(ctx: CreateFunctionContext): LogicalPlan = withOrigin(ctx) { val resources = ctx.resource.asScala.map { resource => - val resourceType = resource.identifier.getText.toLowerCase(Locale.ROOT) + val resourceType = visitIdentifier(resource.identifier).toLowerCase(Locale.ROOT) resourceType match { case "jar" | "file" | "archive" => FunctionResource(FunctionResourceType.fromString(resourceType), string(resource.STRING)) @@ -995,7 +996,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { AlterTableChangeColumnCommand( tableName = visitTableIdentifier(ctx.tableIdentifier), - columnName = ctx.identifier.getText, + columnName = visitIdentifier(ctx.identifier), newColumn = visitColType(ctx.colType)) } @@ -1023,7 +1024,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { } } - orderedIdCtx.identifier.getText + visitIdentifier(orderedIdCtx.identifier) }) } @@ -1071,13 +1072,13 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { val mayebePaths = remainder(ctx.identifier).trim ctx.op.getType match { case SqlBaseParser.ADD => - ctx.identifier.getText.toLowerCase(Locale.ROOT) match { + visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) match { case "file" => AddFileCommand(mayebePaths) case "jar" => AddJarCommand(mayebePaths) case other => operationNotAllowed(s"ADD with resource type '$other'", ctx) } case SqlBaseParser.LIST => - ctx.identifier.getText.toLowerCase(Locale.ROOT) match { + visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) match { case "files" | "file" => if (mayebePaths.length > 0) { ListFilesCommand(mayebePaths.split("\\s+")) @@ -1287,7 +1288,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitGenericFileFormat( ctx: GenericFileFormatContext): CatalogStorageFormat = withOrigin(ctx) { - val source = ctx.identifier.getText + val source = visitIdentifier(ctx.identifier) HiveSerDe.sourceToSerDe(source) match { case Some(s) => CatalogStorageFormat.empty.copy( @@ -1382,7 +1383,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { (rowFormatCtx, createFileFormatCtx.fileFormat) match { case (_, ffTable: TableFileFormatContext) => // OK case (rfSerde: RowFormatSerdeContext, ffGeneric: GenericFileFormatContext) => - ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match { + visitIdentifier(ffGeneric.identifier).toLowerCase(Locale.ROOT) match { case ("sequencefile" | "textfile" | "rcfile") => // OK case fmt => operationNotAllowed( @@ -1390,7 +1391,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { parentCtx) } case (rfDelimited: RowFormatDelimitedContext, ffGeneric: GenericFileFormatContext) => - ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match { + visitIdentifier(ffGeneric.identifier).toLowerCase(Locale.ROOT) match { case "textfile" => // OK case fmt => operationNotAllowed( s"ROW FORMAT DELIMITED is only compatible with 'textfile', not '$fmt'", parentCtx) @@ -1442,7 +1443,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { val userSpecifiedColumns = Option(ctx.identifierCommentList).toSeq.flatMap { icl => icl.identifierComment.asScala.map { ic => - ic.identifier.getText -> Option(ic.STRING).map(string) + visitIdentifier(ic.identifier) -> Option(ic.STRING).map(string) } } From acaab7153723968dbced919ee0992d5a243a3fcf Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Thu, 21 Feb 2019 22:32:22 +0900 Subject: [PATCH 13/18] Fix --- ...sql-reserved-and-non-reserved-key-words.md | 28 +++++++-------- .../spark/sql/catalyst/parser/SqlBase.g4 | 27 +++++++------- .../sql/catalyst/parser/AstBuilder.scala | 36 ++++++------------- .../parser/TableIdentifierParserSuite.scala | 19 ++++++---- 4 files changed, 49 insertions(+), 61 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index 7efd77434a6f2..20e63a161b833 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -22,7 +22,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -112,7 +112,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -179,7 +179,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -209,7 +209,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -237,7 +237,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -246,7 +246,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -254,7 +254,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -276,7 +276,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -309,7 +309,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -319,7 +319,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -341,7 +341,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -427,7 +427,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -447,7 +447,7 @@ A full list of reserved/non-reserved key words is as follows: - + @@ -531,7 +531,7 @@ A full list of reserved/non-reserved key words is as follows: - + diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index aec1911df0e0d..6248069c680b8 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -725,16 +725,13 @@ qualifiedName identifier : strictIdentifier - // We can't move the tokens below into `strictIdentifier` because we need to parse some SQL texts, - // e.g., SELECT * FROM t1 CROSS JOIN t2 - | ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON - | UNION | INTERSECT | EXCEPT | SETMINUS + | {ansi}? ansiReserved + | reserved ; strictIdentifier : IDENTIFIER #unquotedIdentifier | quotedIdentifier #quotedIdentifierAlternative - | {ansi}? ansiReserved #unquotedIdentifier | {ansi}? ansiNonReserved #unquotedIdentifier | {!ansi}? nonReserved #unquotedIdentifier ; @@ -754,10 +751,7 @@ number ; // NOTE: You must follow a rule below when you add a new ANTLR taken in this file: -// - All the ANTLR tokens except for some keywords (ANTI, FULL, INNER, LEFT, SEMI, RIGHT, NATURAL, JOIN, CROSS, ON, -// UNION, INTERSECT, EXCEPT, and SETMINUS) -// = UNION(`ansiReserved`, `ansiNonReserved`) -// = `nonReserved` +// - All the ANTLR tokens = UNION(`ansiReserved`, `ansiNonReserved`) = UNION(`reserved`, `nonReserved`) // // Let's say you add a new token `NEWTOKEN` and this is not reserved regardless of a `spark.sql.parser.ansi.enabled` // value. In this case, you must add a token `NEWTOKEN` in both `ansiNonReserved` and `nonReserved`. @@ -767,11 +761,12 @@ number // that almost all the ANSI SQL standards (SQL-92, SQL-99, SQL-2003, SQL-2008, SQL-2011, and SQL-2016) // and PostgreSQL reserve. ansiReserved - : ALL | AND | ANY | AS | AUTHORIZATION | BOTH | CASE | CAST | CHECK | COLLATE | COLUMN | CONSTRAINT | CREATE - | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DISTINCT | ELSE | END | FALSE | FETCH | FOR - | FOREIGN | FROM | GRANT | GROUP | HAVING | IN | INTO | IS | LEADING | NOT | NULL | ONLY | OR | ORDER | OUTER - | OVERLAPS | PRIMARY | REFERENCES | SELECT | SESSION_USER | SOME | TABLE | THEN | TO | TRAILING | UNIQUE | USER - | USING | WHEN | WHERE | WITH + : ALL | AND | ANTI | ANY | AS | AUTHORIZATION | BOTH | CASE | CAST | CHECK | COLLATE | COLUMN | CONSTRAINT | CREATE + | CROSS | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DISTINCT | ELSE | END | EXCEPT | FALSE + | FETCH | FOR | FOREIGN | FROM | FULL | GRANT | GROUP | HAVING | IN | INNER | INTERSECT | INTO | JOIN | IS + | LEADING | LEFT | NATURAL | NOT | NULL | ON | ONLY | OR | ORDER | OUTER | OVERLAPS | PRIMARY | REFERENCES | RIGHT + | SELECT | SEMI | SESSION_USER | SETMINUS | SOME | TABLE | THEN | TO | TRAILING | UNION | UNIQUE | USER | USING + | WHEN | WHERE | WITH ; // When `spark.sql.parser.ansi.enabled` = true, the `ansiNonReserved` keywords can be used for identifiers. @@ -794,6 +789,10 @@ ansiNonReserved | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE | UNLOCK | UNSET | USE | VALUES | VIEW | WINDOW ; +reserved + : ANTI | CROSS | EXCEPT | FULL | INNER | INTERSECT | JOIN | LEFT | NATURAL | ON | RIGHT | SEMI | SETMINUS | UNION + ; + nonReserved : ADD | AFTER | ALL | ALTER | ANALYZE | AND | ANY | ARCHIVE | ARRAY | AS | ASC | AT | AUTHORIZATION | BETWEEN | BOTH | BUCKET | BUCKETS | BY | CACHE | CASCADE | CASE | CAST | CHANGE | CHECK | CLEAR | CLUSTER | CLUSTERED diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index c9d46644d3f26..fb0302f986f89 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -833,7 +833,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging val tvf = UnresolvedTableValuedFunction( visitIdentifier(func.identifier), func.expression.asScala.map(expression), aliases) - tvf.optionalMap(strictIdentifier(func.tableAlias.strictIdentifier))(aliasPlan) + tvf.optionalMap(func.tableAlias.strictIdentifier)(aliasPlan) } /** @@ -858,7 +858,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } val table = UnresolvedInlineTable(aliases, rows) - table.optionalMap(strictIdentifier(ctx.tableAlias.strictIdentifier))(aliasPlan) + table.optionalMap(ctx.tableAlias.strictIdentifier)(aliasPlan) } /** @@ -898,8 +898,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging /** * Create an alias ([[SubqueryAlias]]) for a [[LogicalPlan]]. */ - private def aliasPlan(alias: String, plan: LogicalPlan): LogicalPlan = { - SubqueryAlias(alias, plan) + private def aliasPlan(alias: ParserRuleContext, plan: LogicalPlan): LogicalPlan = { + SubqueryAlias(alias.getText, plan) } /** @@ -908,7 +908,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ private def mayApplyAliasPlan(tableAlias: TableAliasContext, plan: LogicalPlan): LogicalPlan = { if (tableAlias.strictIdentifier != null) { - val subquery = SubqueryAlias(strictIdentifier(tableAlias.strictIdentifier), plan) + val subquery = SubqueryAlias(tableAlias.strictIdentifier.getText, plan) if (tableAlias.identifierList != null) { val columnNames = visitIdentifierList(tableAlias.identifierList) UnresolvedSubqueryColumnAliases(columnNames, subquery) @@ -1404,29 +1404,13 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } } - private def strictIdentifier(ctx: StrictIdentifierContext): String = { - if (ctx != null) { - val keyword = ctx.getText - ctx match { - case u: UnquotedIdentifierContext => - if (u.ansiReserved() != null) { - throw new ParseException( - s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) - } - case _ => - } - keyword - } else { - null - } - } - override def visitIdentifier(ctx: IdentifierContext): String = withOrigin(ctx) { - if (ctx.strictIdentifier() != null) { - strictIdentifier(ctx.strictIdentifier()) - } else { - ctx.getText + val keyword = ctx.getText + if (ctx.ansiReserved() != null) { + throw new ParseException( + s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) } + keyword } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index ebfc2fecf2f27..f9584b284069f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -129,16 +129,21 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "varying", "versioning", "view", "when", "whenever", "where", "while", "width_bucket", "window", "with", "within", "without", "work", "write", "year", "zone") - val reservedKeywordsInAnsiMode = Set("all", "and", "any", "as", "authorization", "both", "case", - "cast", "check", "collate", "column", "constraint", "create", "current_date", "current_time", - "current_timestamp", "current_user", "distinct", "else", "end", "false", "fetch", "for", - "foreign", "from", "grant", "group", "having", "in", "into", "is", "leading", "not", "null", - "only", "or", "order", "outer", "overlaps", "primary", "references", "select", "session_user", - "some", "table", "then", "to", "trailing", "unique", "user", "using", "when", "where", "with") + val reservedKeywordsInAnsiMode = Set("all", "and", "anti", "any", "as", "authorization", "both", + "case", "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", + "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", + "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", + "intersect", "into", "join", "is", "leading", "left", "natural", "not", "null", "on", "only", + "or", "order", "outer", "overlaps", "primary", "references", "right", "select", "semi", + "session_user", "minus", "some", "table", "then", "to", "trailing", "union", "unique", "user", + "using", "when", "where", "with") val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode - val nonReservedKeywordsInNonAnsiMode = allCandidateKeywords + val reservedKeywordsInNonAnsiMode = Set("anti", "cross", "except", "full", "inner", "intersect", + "join", "left", "natural", "on", "right", "semi", "minus", "union") + + val nonReservedKeywordsInNonAnsiMode = allCandidateKeywords -- reservedKeywordsInNonAnsiMode test("table identifier") { // Regular names. From 7b909b8b16e182defee8ef060dd1289e8f45871a Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Thu, 21 Feb 2019 23:24:30 +0900 Subject: [PATCH 14/18] Update the doc --- docs/sql-reserved-and-non-reserved-key-words.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-key-words.md index 20e63a161b833..53b31793f2e99 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-key-words.md @@ -5,8 +5,11 @@ displayTitle: SQL Reserved/Non-Reserved Key Words --- When `spark.sql.parser.ansi.enabled` is set to true (false by default), some keywords are reserved for Spark SQL. -These reserved key words in this ANSI mode basically follow the ANSI SQL-2011 standard, but it is slightly different between each other. -A full list of reserved/non-reserved key words is as follows: +The reserved keywords cannot be used for identifiers (e.g., table names, view names, column names, column aliases) in a query. +Note that there is one exception; table aliases (e.g., SELECT * FROM t AS t1) always follow the default mode +(see a list below) regardless of the `spark.sql.parser.ansi.enabled` value. These reserved keywords in the ANSI mode basically +follow the ANSI SQL-2011 standard, but it is slightly different between each other. +A full list of the reserved/non-reserved keywords is as follows:
Key WordSpark SQL (ANSI mode)Spark SQL (default mode)SQL-2011
Key WordSpark SQLSQL-2011
ANSI modedefault mode
ABSnon-reservednon-reservedreserved
ABSOLUTEnon-reservednon-reservednon-reserved
ACOSnon-reservednon-reservednon-reserved
ALTERnon-reservednon-reservedreserved
ANALYZEnon-reservednon-reservednon-reserved
ANDreservednon-reservedreserved
ANTInon-reservednon-reservednon-reserved
ANYreservednon-reservedreserved
AREnon-reservednon-reservedreserved
ARCHIVEnon-reservednon-reservednon-reserved
SECTIONnon-reservednon-reservednon-reserved
SEEKnon-reservednon-reservednon-reserved
SELECTreservednon-reservedreserved
SEMInon-reservednon-reservednon-reserved
SENSITIVEnon-reservednon-reservedreserved
SEPARATEDnon-reservednon-reservednon-reserved
SERDEnon-reservednon-reservednon-reserved
SESSIONnon-reservednon-reservednon-reserved
SESSION_USERreservednon-reservedreserved
SETnon-reservednon-reservedreserved
SETMINUSnon-reservednon-reservednon-reserved
SETSnon-reservednon-reservednon-reserved
SHOWnon-reservednon-reservednon-reserved
SIGNALnon-reservednon-reservedreserved
COVAR_POPnon-reservednon-reservedreserved
COVAR_SAMPnon-reservednon-reservedreserved
CREATEreservednon-reservedreserved
CROSSreservednon-reservedreserved
CROSSnon-reservednon-reservedreserved
CUBEnon-reservednon-reservedreserved
CUME_DISTnon-reservednon-reservedreserved
CURRENTnon-reservednon-reservedreserved
ESCAPEnon-reservednon-reservedreserved
ESCAPEDnon-reservednon-reservednon-reserved
EVERYnon-reservednon-reservedreserved
EXCEPTreservednon-reservedreserved
EXCEPTnon-reservednon-reservedreserved
EXCEPTIONnon-reservednon-reservednon-reserved
EXCHANGEnon-reservednon-reservednon-reserved
EXECnon-reservednon-reservedreserved
FRAME_ROWnon-reservednon-reservedreserved
FREEnon-reservednon-reservedreserved
FROMreservednon-reservedreserved
FULLreservednon-reservedreserved
FULLnon-reservednon-reservedreserved
FUNCTIONnon-reservednon-reservedreserved
FUNCTIONSnon-reservednon-reservednon-reserved
FUSIONnon-reservednon-reservednon-reserved
INDEXESnon-reservednon-reservednon-reserved
INITIALnon-reservednon-reservednon-reserved
INITIALLYnon-reservednon-reservednon-reserved
INNERreservednon-reservedreserved
INNERnon-reservednon-reservedreserved
INOUTnon-reservednon-reservedreserved
INPATHnon-reservednon-reservednon-reserved
INPUTnon-reservednon-reservednon-reserved
INSERTnon-reservednon-reservedreserved
INTnon-reservednon-reservedreserved
INTEGERnon-reservednon-reservedreserved
INTERSECTreservednon-reservedreserved
INTERSECTnon-reservednon-reservedreserved
INTERSECTIONnon-reservednon-reservedreserved
INTERVALnon-reservednon-reservedreserved
INTOreservednon-reservedreserved
ISOLATIONnon-reservednon-reservednon-reserved
ITEMSnon-reservednon-reservednon-reserved
ITERATEnon-reservednon-reservedreserved
JOINreservednon-reservedreserved
JOINnon-reservednon-reservedreserved
JSON_ARRAYnon-reservednon-reservednon-reserved
JSON_ARRAYAGGnon-reservednon-reservednon-reserved
JSON_EXISTSnon-reservednon-reservednon-reserved
LEADnon-reservednon-reservedreserved
LEADINGreservednon-reservedreserved
LEAVEnon-reservednon-reservedreserved
LEFTreservednon-reservedreserved
LEFTnon-reservednon-reservedreserved
LEVELnon-reservednon-reservednon-reserved
LIKEnon-reservednon-reservedreserved
LIKE_REGEXnon-reservednon-reservedreserved
MERGEnon-reservednon-reservedreserved
METHODnon-reservednon-reservedreserved
MINnon-reservednon-reservedreserved
MINUSnon-reservednon-reservednon-reserved
MINUTEnon-reservednon-reservedreserved
MODnon-reservednon-reservedreserved
MODIFIESnon-reservednon-reservedreserved
MULTISETnon-reservednon-reservedreserved
NAMESnon-reservednon-reservednon-reserved
NATIONALnon-reservednon-reservedreserved
NATURALreservednon-reservedreserved
NATURALnon-reservednon-reservedreserved
NCHARnon-reservednon-reservedreserved
NCLOBnon-reservednon-reservedreserved
NEWnon-reservednon-reservedreserved
OFFSETnon-reservednon-reservedreserved
OLDnon-reservednon-reservedreserved
OMITnon-reservednon-reservednon-reserved
ONreservednon-reservedreserved
ONnon-reservednon-reservedreserved
ONEnon-reservednon-reservednon-reserved
ONLYreservednon-reservedreserved
OPENnon-reservednon-reservedreserved
RETURNnon-reservednon-reservedreserved
RETURNSnon-reservednon-reservedreserved
REVOKEnon-reservednon-reservedreserved
RIGHTreservednon-reservedreserved
RIGHTnon-reservednon-reservedreserved
RLIKEnon-reservednon-reservednon-reserved
ROLEnon-reservednon-reservednon-reserved
ROLESnon-reservednon-reservednon-reserved
SESSIONnon-reservednon-reservednon-reserved
SESSION_USERreservednon-reservedreserved
SETnon-reservednon-reservedreserved
SETMINUSnon-reservednon-reservednon-reserved
SETSnon-reservednon-reservednon-reserved
SHOWnon-reservednon-reservednon-reserved
SIGNALnon-reservednon-reservedreserved
UNCACHEnon-reservednon-reservednon-reserved
UNDERnon-reservednon-reservednon-reserved
UNDOnon-reservednon-reservedreserved
UNIONreservednon-reservedreserved
UNIONnon-reservednon-reservedreserved
UNIQUEreservednon-reservedreserved
UNKNOWNnon-reservednon-reservedreserved
UNLOCKnon-reservednon-reservednon-reserved
USAGEnon-reservednon-reservednon-reserved
USEnon-reservednon-reservednon-reserved
USERreservednon-reservedreserved
USINGreservedreservedreserved
USINGreservednon-reservedreserved
VALUEnon-reservednon-reservedreserved
VALUESnon-reservednon-reservedreserved
VALUE_OFnon-reservednon-reservedreserved
ALTERnon-reservednon-reservedreserved
ANALYZEnon-reservednon-reservednon-reserved
ANDreservednon-reservedreserved
ANTInon-reservednon-reservednon-reserved
ANTIreservedreservednon-reserved
ANYreservednon-reservedreserved
AREnon-reservednon-reservedreserved
ARCHIVEnon-reservednon-reservednon-reserved
COVAR_POPnon-reservednon-reservedreserved
COVAR_SAMPnon-reservednon-reservedreserved
CREATEreservednon-reservedreserved
CROSSnon-reservednon-reservedreserved
CROSSreservedreservedreserved
CUBEnon-reservednon-reservedreserved
CUME_DISTnon-reservednon-reservedreserved
CURRENTnon-reservednon-reservedreserved
ESCAPEnon-reservednon-reservedreserved
ESCAPEDnon-reservednon-reservednon-reserved
EVERYnon-reservednon-reservedreserved
EXCEPTnon-reservednon-reservedreserved
EXCEPTreservedreservedreserved
EXCEPTIONnon-reservednon-reservednon-reserved
EXCHANGEnon-reservednon-reservednon-reserved
EXECnon-reservednon-reservedreserved
FRAME_ROWnon-reservednon-reservedreserved
FREEnon-reservednon-reservedreserved
FROMreservednon-reservedreserved
FULLnon-reservednon-reservedreserved
FULLreservedreservedreserved
FUNCTIONnon-reservednon-reservedreserved
FUNCTIONSnon-reservednon-reservednon-reserved
FUSIONnon-reservednon-reservednon-reserved
INDEXESnon-reservednon-reservednon-reserved
INITIALnon-reservednon-reservednon-reserved
INITIALLYnon-reservednon-reservednon-reserved
INNERnon-reservednon-reservedreserved
INNERreservedreservedreserved
INOUTnon-reservednon-reservedreserved
INPATHnon-reservednon-reservednon-reserved
INPUTnon-reservednon-reservednon-reserved
INSERTnon-reservednon-reservedreserved
INTnon-reservednon-reservedreserved
INTEGERnon-reservednon-reservedreserved
INTERSECTnon-reservednon-reservedreserved
INTERSECTreservedreservedreserved
INTERSECTIONnon-reservednon-reservedreserved
INTERVALnon-reservednon-reservedreserved
INTOreservednon-reservedreserved
ISOLATIONnon-reservednon-reservednon-reserved
ITEMSnon-reservednon-reservednon-reserved
ITERATEnon-reservednon-reservedreserved
JOINnon-reservednon-reservedreserved
JOINreservedreservedreserved
JSON_ARRAYnon-reservednon-reservednon-reserved
JSON_ARRAYAGGnon-reservednon-reservednon-reserved
JSON_EXISTSnon-reservednon-reservednon-reserved
LEADnon-reservednon-reservedreserved
LEADINGreservednon-reservedreserved
LEAVEnon-reservednon-reservedreserved
LEFTnon-reservednon-reservedreserved
LEFTreservedreservedreserved
LEVELnon-reservednon-reservednon-reserved
LIKEnon-reservednon-reservedreserved
LIKE_REGEXnon-reservednon-reservedreserved
MERGEnon-reservednon-reservedreserved
METHODnon-reservednon-reservedreserved
MINnon-reservednon-reservedreserved
MINUSnon-reservednon-reservednon-reserved
MINUSreservedreservednon-reserved
MINUTEnon-reservednon-reservedreserved
MODnon-reservednon-reservedreserved
MODIFIESnon-reservednon-reservedreserved
MULTISETnon-reservednon-reservedreserved
NAMESnon-reservednon-reservednon-reserved
NATIONALnon-reservednon-reservedreserved
NATURALnon-reservednon-reservedreserved
NATURALreservedreservedreserved
NCHARnon-reservednon-reservedreserved
NCLOBnon-reservednon-reservedreserved
NEWnon-reservednon-reservedreserved
OFFSETnon-reservednon-reservedreserved
OLDnon-reservednon-reservedreserved
OMITnon-reservednon-reservednon-reserved
ONnon-reservednon-reservedreserved
ONreservedreservedreserved
ONEnon-reservednon-reservednon-reserved
ONLYreservednon-reservedreserved
OPENnon-reservednon-reservedreserved
RETURNnon-reservednon-reservedreserved
RETURNSnon-reservednon-reservedreserved
REVOKEnon-reservednon-reservedreserved
RIGHTnon-reservednon-reservedreserved
RIGHTreservedreservedreserved
RLIKEnon-reservednon-reservednon-reserved
ROLEnon-reservednon-reservednon-reserved
ROLESnon-reservednon-reservednon-reserved
SECTIONnon-reservednon-reservednon-reserved
SEEKnon-reservednon-reservednon-reserved
SELECTreservednon-reservedreserved
SEMInon-reservednon-reservednon-reserved
SEMIreservedreservednon-reserved
SENSITIVEnon-reservednon-reservedreserved
SEPARATEDnon-reservednon-reservednon-reserved
SERDEnon-reservednon-reservednon-reserved
UNCACHEnon-reservednon-reservednon-reserved
UNDERnon-reservednon-reservednon-reserved
UNDOnon-reservednon-reservedreserved
UNIONnon-reservednon-reservedreserved
UNIONreservedreservedreserved
UNIQUEreservednon-reservedreserved
UNKNOWNnon-reservednon-reservedreserved
UNLOCKnon-reservednon-reservednon-reserved
From 6d4b5abef57bb23a62e9cdd9ae0ea708078727d0 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Thu, 21 Feb 2019 23:24:39 +0900 Subject: [PATCH 15/18] Fix --- docs/_data/menu-sql.yaml | 4 ++-- ...y-words.md => sql-reserved-and-non-reserved-keywords.md} | 6 +++--- .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 2 +- .../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 3 ++- .../org/apache/spark/sql/execution/SparkSqlParser.scala | 3 ++- 5 files changed, 10 insertions(+), 8 deletions(-) rename docs/{sql-reserved-and-non-reserved-key-words.md => sql-reserved-and-non-reserved-keywords.md} (99%) diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml index 5d321ee0ec11f..9bbb115bcdda5 100644 --- a/docs/_data/menu-sql.yaml +++ b/docs/_data/menu-sql.yaml @@ -70,8 +70,8 @@ url: sql-migration-guide-upgrade.html - text: Compatibility with Apache Hive url: sql-migration-guide-hive-compatibility.html - - text: SQL Reserved/Non-Reserved Key Words - url: sql-reserved-and-non-reserved-key-words.html + - text: SQL Reserved/Non-Reserved Keywords + url: sql-reserved-and-non-reserved-keywords.html - text: Reference url: sql-reference.html subitems: diff --git a/docs/sql-reserved-and-non-reserved-key-words.md b/docs/sql-reserved-and-non-reserved-keywords.md similarity index 99% rename from docs/sql-reserved-and-non-reserved-key-words.md rename to docs/sql-reserved-and-non-reserved-keywords.md index 53b31793f2e99..6671c7825978a 100644 --- a/docs/sql-reserved-and-non-reserved-key-words.md +++ b/docs/sql-reserved-and-non-reserved-keywords.md @@ -1,7 +1,7 @@ --- layout: global -title: SQL Reserved/Non-Reserved Key Words -displayTitle: SQL Reserved/Non-Reserved Key Words +title: SQL Reserved/Non-Reserved Keywords +displayTitle: SQL Reserved/Non-Reserved Keywords --- When `spark.sql.parser.ansi.enabled` is set to true (false by default), some keywords are reserved for Spark SQL. @@ -12,7 +12,7 @@ follow the ANSI SQL-2011 standard, but it is slightly different between each oth A full list of the reserved/non-reserved keywords is as follows:
Key WordSpark SQLSQL-2011
- + diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 6248069c680b8..2ea6301a53c4f 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -726,7 +726,7 @@ qualifiedName identifier : strictIdentifier | {ansi}? ansiReserved - | reserved + | {!ansi}? reserved ; strictIdentifier diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index fb0302f986f89..53f3d49dedf69 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1862,7 +1862,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ override def visitComplexColType(ctx: ComplexColTypeContext): StructField = withOrigin(ctx) { import ctx._ - val structField = StructField(visitIdentifier(identifier), typedVisit(dataType), nullable = true) + val name = visitIdentifier(identifier) + val structField = StructField(name, typedVisit(dataType), nullable = true) if (STRING == null) structField else structField.withComment(string(STRING)) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 69f1701d915fc..851526fb1e6f6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -705,7 +705,8 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitShowFunctions(ctx: ShowFunctionsContext): LogicalPlan = withOrigin(ctx) { import ctx._ - val (user, system) = Option(ctx.identifier).map(i => visitIdentifier(i).toLowerCase(Locale.ROOT)) match { + val (user, system) = Option(ctx.identifier).map { i => + visitIdentifier(i).toLowerCase(Locale.ROOT) } match { case None | Some("all") => (true, true) case Some("system") => (false, true) case Some("user") => (true, false) From 5b4b5ca0628935cb4f70e59aad9dea70e512052b Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Fri, 22 Feb 2019 12:42:03 +0900 Subject: [PATCH 16/18] Fix bugs --- docs/sql-reserved-and-non-reserved-keywords.md | 4 ++-- .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 3 ++- .../sql/catalyst/parser/TableIdentifierParserSuite.scala | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-keywords.md b/docs/sql-reserved-and-non-reserved-keywords.md index 6671c7825978a..6c5b776c19ced 100644 --- a/docs/sql-reserved-and-non-reserved-keywords.md +++ b/docs/sql-reserved-and-non-reserved-keywords.md @@ -5,7 +5,7 @@ displayTitle: SQL Reserved/Non-Reserved Keywords --- When `spark.sql.parser.ansi.enabled` is set to true (false by default), some keywords are reserved for Spark SQL. -The reserved keywords cannot be used for identifiers (e.g., table names, view names, column names, column aliases) in a query. +The reserved keywords cannot be used for identifiers (e.g., table names, view names, column names, column aliases) in queries. Note that there is one exception; table aliases (e.g., SELECT * FROM t AS t1) always follow the default mode (see a list below) regardless of the `spark.sql.parser.ansi.enabled` value. These reserved keywords in the ANSI mode basically follow the ANSI SQL-2011 standard, but it is slightly different between each other. @@ -546,7 +546,7 @@ A full list of the reserved/non-reserved keywords is as follows: - + diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 2ea6301a53c4f..e1f3830c569ee 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -791,6 +791,7 @@ ansiNonReserved reserved : ANTI | CROSS | EXCEPT | FULL | INNER | INTERSECT | JOIN | LEFT | NATURAL | ON | RIGHT | SEMI | SETMINUS | UNION + | USING ; nonReserved @@ -811,7 +812,7 @@ nonReserved | SELECT | SEPARATED | SERDE | SERDEPROPERTIES | SESSION_USER | SET | SETS | SHOW | SKEWED | SOME | SORT | SORTED | START | STATISTICS | STORED | STRATIFY | STRUCT | TABLE | TABLES | TABLESAMPLE | TBLPROPERTIES | TEMPORARY | TERMINATED | THEN | TO | TOUCH | TRAILING | TRANSACTION | TRANSACTIONS | TRANSFORM | TRUE | TRUNCATE | UNARCHIVE - | UNBOUNDED | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | USING | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH + | UNBOUNDED | UNCACHE | UNLOCK | UNIQUE | UNSET | USE | USER | VALUES | VIEW | WHEN | WHERE | WINDOW | WITH ; SELECT: 'SELECT'; diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index f9584b284069f..45482e7a29935 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -141,7 +141,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode val reservedKeywordsInNonAnsiMode = Set("anti", "cross", "except", "full", "inner", "intersect", - "join", "left", "natural", "on", "right", "semi", "minus", "union") + "join", "left", "natural", "on", "right", "semi", "minus", "union", "using") val nonReservedKeywordsInNonAnsiMode = allCandidateKeywords -- reservedKeywordsInNonAnsiMode From 83a107e94b8a191cd0d73e4f6d0b33827361f672 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Fri, 22 Feb 2019 16:09:18 +0900 Subject: [PATCH 17/18] Update docs --- docs/sql-reserved-and-non-reserved-keywords.md | 13 +++++++------ .../spark/sql/catalyst/parser/SqlBase.g4 | 18 ++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/sql-reserved-and-non-reserved-keywords.md b/docs/sql-reserved-and-non-reserved-keywords.md index 6c5b776c19ced..321fb3f00acbd 100644 --- a/docs/sql-reserved-and-non-reserved-keywords.md +++ b/docs/sql-reserved-and-non-reserved-keywords.md @@ -4,12 +4,13 @@ title: SQL Reserved/Non-Reserved Keywords displayTitle: SQL Reserved/Non-Reserved Keywords --- -When `spark.sql.parser.ansi.enabled` is set to true (false by default), some keywords are reserved for Spark SQL. -The reserved keywords cannot be used for identifiers (e.g., table names, view names, column names, column aliases) in queries. -Note that there is one exception; table aliases (e.g., SELECT * FROM t AS t1) always follow the default mode -(see a list below) regardless of the `spark.sql.parser.ansi.enabled` value. These reserved keywords in the ANSI mode basically -follow the ANSI SQL-2011 standard, but it is slightly different between each other. -A full list of the reserved/non-reserved keywords is as follows: +In Spark SQL, there are 2 kinds of keywords: non-reserved and reserved. Non-reserved keywords have a +special meaning only in particular contexts and can be used as identifiers (e.g., table names, view names, +column names, column aliases, table aliases) in other contexts. Reserved keywords can't be used as +table alias, but can be used as other identifiers. + +The list of reserved and non-reserved keywords can change according to the config +`spark.sql.parser.ansi.enabled`, which is false by default.
Key WordSpark SQLSQL-2011
KeywordSpark SQLSQL-2011
ANSI modedefault mode
ABSnon-reservednon-reservedreserved
ABSOLUTEnon-reservednon-reservednon-reserved
USAGEnon-reservednon-reservednon-reserved
USEnon-reservednon-reservednon-reserved
USERreservednon-reservedreserved
USINGreservednon-reservedreserved
USINGreservedreservedreserved
VALUEnon-reservednon-reservedreserved
VALUESnon-reservednon-reservedreserved
VALUE_OFnon-reservednon-reservedreserved
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index e1f3830c569ee..69deeb518d6e5 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -726,7 +726,7 @@ qualifiedName identifier : strictIdentifier | {ansi}? ansiReserved - | {!ansi}? reserved + | {!ansi}? defaultReserved ; strictIdentifier @@ -751,15 +751,14 @@ number ; // NOTE: You must follow a rule below when you add a new ANTLR taken in this file: -// - All the ANTLR tokens = UNION(`ansiReserved`, `ansiNonReserved`) = UNION(`reserved`, `nonReserved`) +// - All the ANTLR tokens = UNION(`ansiReserved`, `ansiNonReserved`) = UNION(`defaultReserved`, `nonReserved`) // // Let's say you add a new token `NEWTOKEN` and this is not reserved regardless of a `spark.sql.parser.ansi.enabled` // value. In this case, you must add a token `NEWTOKEN` in both `ansiNonReserved` and `nonReserved`. -// A list of the reserved keywords below in Spark SQL. These keywords cannot be used for identifiers -// when `spark.sql.parser.ansi.enabled` = true. Currently, we only reserve the ANSI keywords -// that almost all the ANSI SQL standards (SQL-92, SQL-99, SQL-2003, SQL-2008, SQL-2011, and SQL-2016) -// and PostgreSQL reserve. +// The list of the reserved keywords when `spark.sql.parser.ansi.enabled` is true. Currently, we only reserve +// the ANSI keywords that almost all the ANSI SQL standards (SQL-92, SQL-99, SQL-2003, SQL-2008, SQL-2011, +// and SQL-2016) and PostgreSQL reserve. ansiReserved : ALL | AND | ANTI | ANY | AS | AUTHORIZATION | BOTH | CASE | CAST | CHECK | COLLATE | COLUMN | CONSTRAINT | CREATE | CROSS | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | DISTINCT | ELSE | END | EXCEPT | FALSE @@ -769,9 +768,8 @@ ansiReserved | WHEN | WHERE | WITH ; -// When `spark.sql.parser.ansi.enabled` = true, the `ansiNonReserved` keywords can be used for identifiers. -// Otherwise (`spark.sql.parser.ansi.enabled` = false), we follow the existing Spark SQL behaviour until v3.0: -// the `nonReserved` keywords can be used instead. + +// The list of the non-reserved keywords when `spark.sql.parser.ansi.enabled` is true. ansiNonReserved : ADD | AFTER | ALTER | ANALYZE | ARCHIVE | ARRAY | ASC | AT | BETWEEN | BUCKET | BUCKETS | BY | CACHE | CASCADE | CHANGE | CLEAR | CLUSTER | CLUSTERED | CODEGEN | COLLECTION | COLUMNS | COMMENT | COMMIT | COMPACT | COMPACTIONS @@ -789,7 +787,7 @@ ansiNonReserved | TRUE | TRUNCATE | UNARCHIVE | UNBOUNDED | UNCACHE | UNLOCK | UNSET | USE | VALUES | VIEW | WINDOW ; -reserved +defaultReserved : ANTI | CROSS | EXCEPT | FULL | INNER | INTERSECT | JOIN | LEFT | NATURAL | ON | RIGHT | SEMI | SETMINUS | UNION | USING ; From d1ab4f438b296c72953e8609510a628467c3fc95 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Fri, 22 Feb 2019 16:18:28 +0900 Subject: [PATCH 18/18] Revert some stuffs --- .../sql/catalyst/parser/AstBuilder.scala | 42 +++--- .../sql/catalyst/parser/ParseDriver.scala | 2 +- .../parser/TableIdentifierParserSuite.scala | 122 +----------------- .../spark/sql/execution/SparkSqlParser.scala | 34 +++-- 4 files changed, 39 insertions(+), 161 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 53f3d49dedf69..24bbe116ad897 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -271,7 +271,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitPartitionSpec( ctx: PartitionSpecContext): Map[String, Option[String]] = withOrigin(ctx) { val parts = ctx.partitionVal.asScala.map { pVal => - val name = visitIdentifier(pVal.identifier) + val name = pVal.identifier.getText val value = Option(pVal.constant).map(visitStringConstant) name -> value } @@ -570,7 +570,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging // Collect all window specifications defined in the WINDOW clause. val baseWindowMap = ctx.namedWindow.asScala.map { wCtx => - (visitIdentifier(wCtx.identifier), typedVisit[WindowSpec](wCtx.windowSpec)) + (wCtx.identifier.getText, typedVisit[WindowSpec](wCtx.windowSpec)) }.toMap // Handle cases like @@ -645,11 +645,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging .flatMap(_.namedExpression.asScala) .map(typedVisit[Expression]) val pivotColumn = if (ctx.pivotColumn.identifiers.size == 1) { - UnresolvedAttribute.quoted(visitIdentifier(ctx.pivotColumn.identifier)) + UnresolvedAttribute.quoted(ctx.pivotColumn.identifier.getText) } else { CreateStruct( ctx.pivotColumn.identifiers.asScala.map( - identifier => UnresolvedAttribute.quoted(visitIdentifier(identifier)))) + identifier => UnresolvedAttribute.quoted(identifier.getText))) } val pivotValues = ctx.pivotValues.asScala.map(visitPivotValue) Pivot(None, pivotColumn, pivotValues, aggregates, query) @@ -661,7 +661,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitPivotValue(ctx: PivotValueContext): Expression = withOrigin(ctx) { val e = expression(ctx.expression) if (ctx.identifier != null) { - Alias(e, visitIdentifier(ctx.identifier))() + Alias(e, ctx.identifier.getText)() } else { e } @@ -832,7 +832,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } val tvf = UnresolvedTableValuedFunction( - visitIdentifier(func.identifier), func.expression.asScala.map(expression), aliases) + func.identifier.getText, func.expression.asScala.map(expression), aliases) tvf.optionalMap(func.tableAlias.strictIdentifier)(aliasPlan) } @@ -931,7 +931,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a Sequence of Strings for an identifier list. */ override def visitIdentifierSeq(ctx: IdentifierSeqContext): Seq[String] = withOrigin(ctx) { - ctx.identifier.asScala.map(visitIdentifier) + ctx.identifier.asScala.map(_.getText) } /* ******************************************************************************************** @@ -942,7 +942,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ override def visitTableIdentifier( ctx: TableIdentifierContext): TableIdentifier = withOrigin(ctx) { - TableIdentifier(visitIdentifier(ctx.table), Option(ctx.db).map(_.getText)) + TableIdentifier(ctx.table.getText, Option(ctx.db).map(_.getText)) } /** @@ -974,7 +974,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Both un-targeted (global) and targeted aliases are supported. */ override def visitStar(ctx: StarContext): Expression = withOrigin(ctx) { - UnresolvedStar(Option(ctx.qualifiedName()).map(_.identifier.asScala.map(visitIdentifier))) + UnresolvedStar(Option(ctx.qualifiedName()).map(_.identifier.asScala.map(_.getText))) } /** @@ -984,7 +984,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging override def visitNamedExpression(ctx: NamedExpressionContext): Expression = withOrigin(ctx) { val e = expression(ctx.expression) if (ctx.identifier != null) { - Alias(e, visitIdentifier(ctx.identifier))() + Alias(e, ctx.identifier.getText)() } else if (ctx.identifierList != null) { MultiAlias(e, visitIdentifierList(ctx.identifierList)) } else { @@ -1326,7 +1326,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a function database (optional) and name pair. */ protected def visitFunctionName(ctx: QualifiedNameContext): FunctionIdentifier = { - ctx.identifier().asScala.map(visitIdentifier) match { + ctx.identifier().asScala.map(_.getText) match { case Seq(db, fn) => FunctionIdentifier(fn, Option(db)) case Seq(fn) => FunctionIdentifier(fn, None) case other => throw new ParseException(s"Unsupported function name '${ctx.getText}'", ctx) @@ -1350,7 +1350,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a reference to a window frame, i.e. [[WindowSpecReference]]. */ override def visitWindowRef(ctx: WindowRefContext): WindowSpecReference = withOrigin(ctx) { - WindowSpecReference(visitIdentifier(ctx.identifier)) + WindowSpecReference(ctx.identifier.getText) } /** @@ -1404,15 +1404,6 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } } - override def visitIdentifier(ctx: IdentifierContext): String = withOrigin(ctx) { - val keyword = ctx.getText - if (ctx.ansiReserved() != null) { - throw new ParseException( - s"'$keyword' is reserved and you cannot use this keyword as an identifier.", ctx) - } - keyword - } - /** * Create a [[CreateStruct]] expression. */ @@ -1560,7 +1551,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ override def visitTypeConstructor(ctx: TypeConstructorContext): Literal = withOrigin(ctx) { val value = string(ctx.STRING) - val valueType = visitIdentifier(ctx.identifier).toUpperCase(Locale.ROOT) + val valueType = ctx.identifier.getText.toUpperCase(Locale.ROOT) def toLiteral[T](f: UTF8String => Option[T], t: DataType): Literal = { f(UTF8String.fromString(value)).map(Literal(_, t)).getOrElse { throw new ParseException(s"Cannot parse the $valueType value: $value", ctx) @@ -1764,7 +1755,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Resolve/create a primitive type. */ override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType = withOrigin(ctx) { - val dataType = visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) + val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT) (dataType, ctx.INTEGER_VALUE().asScala.toList) match { case ("boolean", Nil) => BooleanType case ("tinyint" | "byte", Nil) => ByteType @@ -1836,7 +1827,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } StructField( - visitIdentifier(identifier), + identifier.getText, cleanedDataType, nullable = true, builder.build()) @@ -1862,8 +1853,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging */ override def visitComplexColType(ctx: ComplexColTypeContext): StructField = withOrigin(ctx) { import ctx._ - val name = visitIdentifier(identifier) - val structField = StructField(name, typedVisit(dataType), nullable = true) + val structField = StructField(identifier.getText, typedVisit(dataType), nullable = true) if (STRING == null) structField else structField.withComment(string(STRING)) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala index 1d5de0a353304..446cb0964188a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala @@ -133,7 +133,7 @@ class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser { /** For test-only. */ object CatalystSqlParser extends AbstractSqlParser { - val astBuilder = new AstBuilder(SQLConf.get) + val astBuilder = new AstBuilder(new SQLConf()) } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index 45482e7a29935..ff0de0fb7c1f0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -18,14 +18,12 @@ package org.apache.spark.sql.catalyst.parser import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.plans.SQLHelper -import org.apache.spark.sql.internal.SQLConf -class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { +class TableIdentifierParserSuite extends SparkFunSuite { import CatalystSqlParser._ // Add "$elem$", "$value$" & "$key$" - val hiveNonReservedKeywords = Seq("add", "admin", "after", "analyze", "archive", "asc", "before", + val hiveNonReservedKeyword = Array("add", "admin", "after", "analyze", "archive", "asc", "before", "bucket", "buckets", "cascade", "change", "cluster", "clustered", "clusterstatus", "collection", "columns", "comment", "compact", "compactions", "compute", "concatenate", "continue", "cost", "data", "day", "databases", "datetime", "dbproperties", "deferred", "defined", "delimited", @@ -55,96 +53,10 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "bigint", "binary", "boolean", "current_date", "current_timestamp", "date", "double", "float", "int", "smallint", "timestamp", "at", "position", "both", "leading", "trailing", "extract") - val hiveStrictNonReservedKeywords = Seq("anti", "full", "inner", "left", "semi", "right", + val hiveStrictNonReservedKeyword = Seq("anti", "full", "inner", "left", "semi", "right", "natural", "union", "intersect", "except", "database", "on", "join", "cross", "select", "from", "where", "having", "from", "to", "table", "with", "not") - // All the keywords in `docs/sql-reserved-and-non-reserved-key-words.md` are listed below: - val allCandidateKeywords = Set("abs", "absolute", "acos", "action", "add", "after", - "all", "allocate", "alter", "analyze", "and", "anti", "any", "archive", "are", "array", - "array_agg", "array_max_cardinality", "as", "asc", "asensitive", "asin", "assertion", - "asymmetric", "at", "atan", "atomic", "authorization", "avg", "before", "begin", "begin_frame", - "begin_partition", "between", "bigint", "binary", "bit", "bit_length", "blob", "boolean", - "both", "breadth", "bucket", "buckets", "by", "cache", "call", "called", "cardinality", - "cascade", "cascaded", "case", "cast", "catalog", "ceil", "ceiling", "change", "char", - "char_length", "character", "character_length", "check", "classifier", "clear", "clob", "close", - "cluster", "clustered", "coalesce", "codegen", "collate", "collation", "collect", "collection", - "column", "columns", "comment", "commit", "compact", "compactions", "compute", "concatenate", - "condition", "connect", "connection", "constraint", "constraints", "constructor", "contains", - "continue", "convert", "copy", "corr", "corresponding", "cos", "cosh", "cost", "count", - "covar_pop", "covar_samp", "create", "cross", "cube", "cume_dist", "current", "current_catalog", - "current_date", "current_default_transform_group", "current_path", "current_role", - "current_row", "current_schema", "current_time", "current_timestamp", - "current_transform_group_for_type", "current_user", "cursor", "cycle", "data", "database", - "databases", "date", "day", "dbproperties", "deallocate", "dec", "decfloat", "decimal", - "declare", "default", "deferrable", "deferred", "define", "defined", "delete", "delimited", - "dense_rank", "depth", "deref", "desc", "describe", "descriptor", "deterministic", "dfs", - "diagnostics", "directories", "directory", "disconnect", "distinct", "distribute", "div", "do", - "domain", "double", "drop", "dynamic", "each", "element", "else", "elseif", "empty", "end", - "end_frame", "end_partition", "equals", "escape", "escaped", "every", "except", "exception", - "exchange", "exec", "execute", "exists", "exit", "exp", "explain", "export", "extended", - "external", "extract", "false", "fetch", "fields", "fileformat", "filter", "first", - "first_value", "float", "following", "for", "foreign", "format", "formatted", "found", - "frame_row", "free", "from", "full", "function", "functions", "fusion", "general", "get", - "global", "go", "goto", "grant", "group", "grouping", "groups", "handler", "having", "hold", - "hour", "identity", "if", "ignore", "immediate", "import", "in", "index", "indexes", - "indicator", "initial", "initially", "inner", "inout", "inpath", "input", "inputformat", - "insensitive", "insert", "int", "integer", "intersect", "intersection", "interval", "into", - "is", "isolation", "items", "iterate", "join", "json_array", "json_arrayagg", "json_exists", - "json_object", "json_objectagg", "json_query", "json_table", "json_table_primitive", - "json_value", "key", "keys", "lag", "language", "large", "last", "last_value", "lateral", - "lazy", "lead", "leading", "leave", "left", "level", "like", "like_regex", "limit", "lines", - "list", "listagg", "ln", "load", "local", "localtime", "localtimestamp", "location", - "locator", "lock", "locks", "log", "log10", "logical", "loop", "lower", "macro", "map", - "match", "match_number", "match_recognize", "matches", "max", "member", "merge", "method", - "min", "minus", "minute", "mod", "modifies", "module", "month", "msck", "multiset", "names", - "national", "natural", "nchar", "nclob", "new", "next", "no", "none", "normalize", - "not", "nth_value", "ntile", "null", "nullif", "nulls", "numeric", "object", - "occurrences_regex", "octet_length", "of", "offset", "old", "omit", "on", "one", "only", "open", - "option", "options", "or", "order", "ordinality", "out", "outer", "output", "outputformat", - "over", "overlaps", "overlay", "overwrite", "pad", "parameter", "partial", "partition", - "partitioned", "partitions", "path", "pattern", "per", "percent", "percent_rank", - "percentile_cont", "percentile_disc", "percentlit", "period", "pivot", "portion", "power", - "precedes", "preceding", "precision", "prepare", "preserve", "primary", "principals", "prior", - "privileges", "procedure", "ptf", "public", "purge", "range", "rank", "read", "reads", "real", - "recordreader", "recordwriter", "recover", "recursive", "reduce", "ref", "references", - "referencing", "refresh", "regr_avgx", "regr_avgy", "regr_count", "regr_intercept", "regr_r2", - "regr_slope", "regr_sxx", "regr_sxy", "regr_syy", "relative", "release", "rename", "repair", - "repeat", "replace", "reset", "resignal", "restrict", "result", "return", "returns", "revoke", - "right", "rlike", "role", "roles", "rollback", "rollup", "routine", "row", "row_number", "rows", - "running", "savepoint", "schema", "scope", "scroll", "search", "second", "section", "seek", - "select", "semi", "sensitive", "separated", "serde", "serdeproperties", "session", - "session_user", "set", "sets", "show", "signal", "similar", "sin", "sinh", "size", - "skewed", "skip", "smallint", "some", "sort", "sorted", "space", "specific", "specifictype", - "sql", "sqlcode", "sqlerror", "sqlexception", "sqlstate", "sqlwarning", "sqrt", "start", - "state", "static", "statistics", "stddev_pop", "stddev_samp", "stored", "stratify", "struct", - "submultiset", "subset", "substring", "substring_regex", "succeeds", "sum", "symmetric", - "system", "system_time", "system_user", "table", "tables", "tablesample", "tan", "tanh", - "tblproperties", "temporary", "terminated", "then", "time", "timestamp", "timezone_hour", - "timezone_minute", "to", "touch", "trailing", "transaction", "transactions", "transform", - "translate", "translate_regex", "translation", "treat", "trigger", "trim", "trim_array", "true", - "truncate", "uescape", "unarchive", "unbounded", "uncache", "under", "undo", "union", "unique", - "unknown", "unlock", "unnest", "unset", "until", "update", "upper", "usage", "use", "user", - "using", "value", "value_of", "values", "var_pop", "var_samp", "varbinary", "varchar", - "varying", "versioning", "view", "when", "whenever", "where", "while", "width_bucket", "window", - "with", "within", "without", "work", "write", "year", "zone") - - val reservedKeywordsInAnsiMode = Set("all", "and", "anti", "any", "as", "authorization", "both", - "case", "cast", "check", "collate", "column", "constraint", "create", "cross", "current_date", - "current_time", "current_timestamp", "current_user", "distinct", "else", "end", "except", - "false", "fetch", "for", "foreign", "from", "full", "grant", "group", "having", "in", "inner", - "intersect", "into", "join", "is", "leading", "left", "natural", "not", "null", "on", "only", - "or", "order", "outer", "overlaps", "primary", "references", "right", "select", "semi", - "session_user", "minus", "some", "table", "then", "to", "trailing", "union", "unique", "user", - "using", "when", "where", "with") - - val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode - - val reservedKeywordsInNonAnsiMode = Set("anti", "cross", "except", "full", "inner", "intersect", - "join", "left", "natural", "on", "right", "semi", "minus", "union", "using") - - val nonReservedKeywordsInNonAnsiMode = allCandidateKeywords -- reservedKeywordsInNonAnsiMode - test("table identifier") { // Regular names. assert(TableIdentifier("q") === parseTableIdentifier("q")) @@ -164,27 +76,9 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { assert(TableIdentifier("x.y.z", None) === parseTableIdentifier("`x.y.z`")) } - test("table identifier - reserved/non-reserved keywords if ANSI mode enabled") { - withSQLConf(SQLConf.ANSI_SQL_PARSER.key -> "true") { - reservedKeywordsInAnsiMode.foreach { keyword => - val errMsg = intercept[ParseException] { - parseTableIdentifier(keyword) - }.getMessage - assert(errMsg.contains(s"'$keyword' is reserved and you cannot use this keyword " + - "as an identifier.")) - assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) - assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) - } - nonReservedKeywordsInAnsiMode.foreach { keyword => - assert(TableIdentifier(keyword) === parseTableIdentifier(s"$keyword")) - assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.$keyword")) - } - } - } - test("table identifier - strict keywords") { // SQL Keywords. - hiveStrictNonReservedKeywords.foreach { keyword => + hiveStrictNonReservedKeyword.foreach { keyword => assert(TableIdentifier(keyword) === parseTableIdentifier(keyword)) assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`")) assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`")) @@ -192,12 +86,8 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { } test("table identifier - non reserved keywords") { - nonReservedKeywordsInNonAnsiMode.foreach { nonReserved => - assert(TableIdentifier(nonReserved) === parseTableIdentifier(nonReserved)) - } - - // Additionally checks if Hive keywords can be allowed. - hiveNonReservedKeywords.foreach { nonReserved => + // Hive keywords are allowed. + hiveNonReservedKeyword.foreach { nonReserved => assert(TableIdentifier(nonReserved) === parseTableIdentifier(nonReserved)) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 851526fb1e6f6..8deb55b00a9d3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -116,9 +116,8 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { } } if (ctx.identifier != null && - visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) != "noscan") { - throw new ParseException("Expected `NOSCAN` instead of " + - s"`${visitIdentifier(ctx.identifier)}`", ctx) + ctx.identifier.getText.toLowerCase(Locale.ROOT) != "noscan") { + throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`", ctx) } val table = visitTableIdentifier(ctx.tableIdentifier) @@ -638,7 +637,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitCreateDatabase(ctx: CreateDatabaseContext): LogicalPlan = withOrigin(ctx) { CreateDatabaseCommand( - visitIdentifier(ctx.identifier), + ctx.identifier.getText, ctx.EXISTS != null, Option(ctx.locationSpec).map(visitLocationSpec), Option(ctx.comment).map(string), @@ -656,7 +655,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { override def visitSetDatabaseProperties( ctx: SetDatabasePropertiesContext): LogicalPlan = withOrigin(ctx) { AlterDatabasePropertiesCommand( - visitIdentifier(ctx.identifier), + ctx.identifier.getText, visitPropertyKeyValues(ctx.tablePropertyList)) } @@ -669,7 +668,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { * }}} */ override def visitDropDatabase(ctx: DropDatabaseContext): LogicalPlan = withOrigin(ctx) { - DropDatabaseCommand(visitIdentifier(ctx.identifier), ctx.EXISTS != null, ctx.CASCADE != null) + DropDatabaseCommand(ctx.identifier.getText, ctx.EXISTS != null, ctx.CASCADE != null) } /** @@ -681,7 +680,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { * }}} */ override def visitDescribeDatabase(ctx: DescribeDatabaseContext): LogicalPlan = withOrigin(ctx) { - DescribeDatabaseCommand(visitIdentifier(ctx.identifier), ctx.EXTENDED != null) + DescribeDatabaseCommand(ctx.identifier.getText, ctx.EXTENDED != null) } /** @@ -705,8 +704,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitShowFunctions(ctx: ShowFunctionsContext): LogicalPlan = withOrigin(ctx) { import ctx._ - val (user, system) = Option(ctx.identifier).map { i => - visitIdentifier(i).toLowerCase(Locale.ROOT) } match { + val (user, system) = Option(ctx.identifier).map(_.getText.toLowerCase(Locale.ROOT)) match { case None | Some("all") => (true, true) case Some("system") => (false, true) case Some("user") => (true, false) @@ -736,7 +734,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitCreateFunction(ctx: CreateFunctionContext): LogicalPlan = withOrigin(ctx) { val resources = ctx.resource.asScala.map { resource => - val resourceType = visitIdentifier(resource.identifier).toLowerCase(Locale.ROOT) + val resourceType = resource.identifier.getText.toLowerCase(Locale.ROOT) resourceType match { case "jar" | "file" | "archive" => FunctionResource(FunctionResourceType.fromString(resourceType), string(resource.STRING)) @@ -997,7 +995,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { AlterTableChangeColumnCommand( tableName = visitTableIdentifier(ctx.tableIdentifier), - columnName = visitIdentifier(ctx.identifier), + columnName = ctx.identifier.getText, newColumn = visitColType(ctx.colType)) } @@ -1025,7 +1023,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { } } - visitIdentifier(orderedIdCtx.identifier) + orderedIdCtx.identifier.getText }) } @@ -1073,13 +1071,13 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { val mayebePaths = remainder(ctx.identifier).trim ctx.op.getType match { case SqlBaseParser.ADD => - visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) match { + ctx.identifier.getText.toLowerCase(Locale.ROOT) match { case "file" => AddFileCommand(mayebePaths) case "jar" => AddJarCommand(mayebePaths) case other => operationNotAllowed(s"ADD with resource type '$other'", ctx) } case SqlBaseParser.LIST => - visitIdentifier(ctx.identifier).toLowerCase(Locale.ROOT) match { + ctx.identifier.getText.toLowerCase(Locale.ROOT) match { case "files" | "file" => if (mayebePaths.length > 0) { ListFilesCommand(mayebePaths.split("\\s+")) @@ -1289,7 +1287,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { */ override def visitGenericFileFormat( ctx: GenericFileFormatContext): CatalogStorageFormat = withOrigin(ctx) { - val source = visitIdentifier(ctx.identifier) + val source = ctx.identifier.getText HiveSerDe.sourceToSerDe(source) match { case Some(s) => CatalogStorageFormat.empty.copy( @@ -1384,7 +1382,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { (rowFormatCtx, createFileFormatCtx.fileFormat) match { case (_, ffTable: TableFileFormatContext) => // OK case (rfSerde: RowFormatSerdeContext, ffGeneric: GenericFileFormatContext) => - visitIdentifier(ffGeneric.identifier).toLowerCase(Locale.ROOT) match { + ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match { case ("sequencefile" | "textfile" | "rcfile") => // OK case fmt => operationNotAllowed( @@ -1392,7 +1390,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { parentCtx) } case (rfDelimited: RowFormatDelimitedContext, ffGeneric: GenericFileFormatContext) => - visitIdentifier(ffGeneric.identifier).toLowerCase(Locale.ROOT) match { + ffGeneric.identifier.getText.toLowerCase(Locale.ROOT) match { case "textfile" => // OK case fmt => operationNotAllowed( s"ROW FORMAT DELIMITED is only compatible with 'textfile', not '$fmt'", parentCtx) @@ -1444,7 +1442,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { val userSpecifiedColumns = Option(ctx.identifierCommentList).toSeq.flatMap { icl => icl.identifierComment.asScala.map { ic => - visitIdentifier(ic.identifier) -> Option(ic.STRING).map(string) + ic.identifier.getText -> Option(ic.STRING).map(string) } }
KeywordSpark SQLSQL-2011