From 49ee1d7a57c5db7cc937ff3b39cd43d78c3763c2 Mon Sep 17 00:00:00 2001 From: Alex Kyllo Date: Fri, 3 Jan 2020 18:20:48 -0800 Subject: [PATCH 1/7] tokenizer test: can tokenize TOP keyword --- src/dialect/keywords.rs | 3 ++- src/tokenizer.rs | 26 ++++++++++++++++++++++++++ tests/sqlparser_mssql.rs | 7 +++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/dialect/keywords.rs b/src/dialect/keywords.rs index c083c0692..9795f2af3 100644 --- a/src/dialect/keywords.rs +++ b/src/dialect/keywords.rs @@ -374,6 +374,7 @@ define_keywords!( TIMEZONE_HOUR, TIMEZONE_MINUTE, TO, + TOP, TRAILING, TRANSACTION, TRANSLATE, @@ -426,7 +427,7 @@ define_keywords!( /// can be parsed unambiguously without looking ahead. pub const RESERVED_FOR_TABLE_ALIAS: &[&str] = &[ // Reserved as both a table and a column alias: - WITH, SELECT, WHERE, GROUP, HAVING, ORDER, LIMIT, OFFSET, FETCH, UNION, EXCEPT, INTERSECT, + WITH, SELECT, WHERE, GROUP, HAVING, ORDER, TOP, LIMIT, OFFSET, FETCH, UNION, EXCEPT, INTERSECT, // Reserved only as a table alias in the `FROM`/`JOIN` clauses: ON, JOIN, INNER, CROSS, FULL, LEFT, RIGHT, NATURAL, USING, // for MSSQL-specific OUTER APPLY (seems reserved in most dialects) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 62d534895..8cc63b5c8 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -522,6 +522,7 @@ fn peeking_take_while( #[cfg(test)] mod tests { use super::super::dialect::GenericDialect; + use super::super::dialect::MsSqlDialect; use super::*; #[test] @@ -782,6 +783,31 @@ mod tests { compare(expected, tokens); } + #[test] + fn tokenize_mssql_top() { + let sql = "SELECT TOP 5 [bar] FROM foo"; + //let select = ms_and_generic().verified_query(sql); + //assert_eq!(Some(Expr::Value(number("5"))), select.limit); + let dialect = MsSqlDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let expected = vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::make_keyword("TOP"), + Token::Whitespace(Whitespace::Space), + Token::Number(String::from("5")), + Token::Whitespace(Whitespace::Space), + Token::make_word("bar", Some('[')), + Token::Whitespace(Whitespace::Space), + Token::make_keyword("FROM"), + Token::Whitespace(Whitespace::Space), + Token::make_word("foo", None), + ]; + compare(expected, tokens); + } + + fn compare(expected: Vec, actual: Vec) { //println!("------------------------------"); //println!("tokens = {:?}", actual); diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index b5170e208..c532fecfe 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -68,6 +68,13 @@ fn parse_mssql_apply_join() { ); } +#[test] +fn parse_mssql_top() { + let sql = "SELECT TOP 5 [bar] FROM foo"; + let select = ms_and_generic().verified_query(sql); + assert_eq!(Some(Expr::Value(number("5"))), select.limit); +} + fn ms() -> TestedDialects { TestedDialects { dialects: vec![Box::new(MsSqlDialect {})], From 38f0b55e7bfb8e97e85c23e3a769e852d63990c4 Mon Sep 17 00:00:00 2001 From: Alex Kyllo Date: Fri, 3 Jan 2020 23:26:39 -0800 Subject: [PATCH 2/7] add support for MSSQL SELECT TOP (N) [PERCENT] [WITH TIES] syntax --- src/ast/query.rs | 19 +++++++++++++------ src/parser.rs | 30 ++++++++++++++++++++++++++++++ src/tokenizer.rs | 3 --- tests/sqlparser_mssql.rs | 37 ++++++++++++++++++++++++++++++++++--- 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/ast/query.rs b/src/ast/query.rs index 656f7f14b..0e6e50ac6 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -114,6 +114,10 @@ impl fmt::Display for SetOperator { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Select { pub distinct: bool, + /// top and percent are MSSQL only + pub top: Option, + pub percent: bool, + pub with_ties: bool, /// projection expressions pub projection: Vec, /// FROM @@ -128,12 +132,15 @@ pub struct Select { impl fmt::Display for Select { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "SELECT{} {}", - if self.distinct { " DISTINCT" } else { "" }, - display_comma_separated(&self.projection) - )?; + write!(f, "SELECT{}", if self.distinct { " DISTINCT" } else { "" })?; + if let Some(ref top) = self.top { + write!(f, + " TOP ({}){}{}", + top, + if self.percent { " PERCENT" } else { "" }, + if self.with_ties { " WITH TIES"} else { "" })?; + } + write!(f, " {}", display_comma_separated(&self.projection))?; if !self.from.is_empty() { write!(f, " FROM {}", display_comma_separated(&self.from))?; } diff --git a/src/parser.rs b/src/parser.rs index cbdcaba09..09f1c03c5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1561,6 +1561,21 @@ impl Parser { if all && distinct { return parser_err!("Cannot specify both ALL and DISTINCT in SELECT"); } + + let top = if self.parse_keyword("TOP") { + self.parse_top()? + } else { + None + }; + + let percent = self.parse_keyword("PERCENT"); + + let with_ties = self.parse_keywords(vec!["WITH", "TIES"]); + + if top.is_none() && percent { + return parser_err!("Cannot specify PERCENT without TOP in SELECT"); + } + let projection = self.parse_comma_separated(Parser::parse_select_item)?; // Note that for keywords to be properly handled here, they need to be @@ -1594,6 +1609,9 @@ impl Parser { Ok(Select { distinct, + top, + percent, + with_ties, projection, from, selection, @@ -1940,6 +1958,18 @@ impl Parser { Ok(OrderByExpr { expr, asc }) } + /// Parse a TOP clause, MSSQL equivalent of LIMIT, + /// that follows after SELECT [DISTINCT]. + pub fn parse_top(&mut self) -> Result, ParserError> { + if self.consume_token(&Token::LParen) { + let expr = self.parse_expr()?; + self.expect_token(&Token::RParen)?; + Ok(Some(expr)) + } else { + Ok(Some(Expr::Value(self.parse_number_value()?))) + } + } + /// Parse a LIMIT clause pub fn parse_limit(&mut self) -> Result, ParserError> { if self.parse_keyword("ALL") { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 8cc63b5c8..96c9535ea 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -786,8 +786,6 @@ mod tests { #[test] fn tokenize_mssql_top() { let sql = "SELECT TOP 5 [bar] FROM foo"; - //let select = ms_and_generic().verified_query(sql); - //assert_eq!(Some(Expr::Value(number("5"))), select.limit); let dialect = MsSqlDialect {}; let mut tokenizer = Tokenizer::new(&dialect, sql); let tokens = tokenizer.tokenize().unwrap(); @@ -807,7 +805,6 @@ mod tests { compare(expected, tokens); } - fn compare(expected: Vec, actual: Vec) { //println!("------------------------------"); //println!("tokens = {:?}", actual); diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index c532fecfe..ee05a4810 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -68,11 +68,42 @@ fn parse_mssql_apply_join() { ); } +#[test] +fn parse_mssql_top_paren() { + let sql = "SELECT TOP (5) * FROM foo"; + let select = ms_and_generic().verified_only_select(sql); + assert_eq!(Some(Expr::Value(number("5"))), select.top); + assert!(!select.percent); +} + +#[test] +fn parse_mssql_top_percent() { + let sql = "SELECT TOP (5) PERCENT * FROM foo"; + let select = ms_and_generic().verified_only_select(sql); + assert_eq!(Some(Expr::Value(number("5"))), select.top); + assert!(select.percent); +} + +#[test] +fn parse_mssql_top_with_ties() { + let sql = "SELECT TOP (5) WITH TIES * FROM foo"; + let select = ms_and_generic().verified_only_select(sql); + assert_eq!(Some(Expr::Value(number("5"))), select.top); + assert!(select.with_ties); +} + +#[test] +fn parse_mssql_top_percent_with_ties() { + let sql = "SELECT TOP (5) PERCENT WITH TIES * FROM foo"; + let select = ms_and_generic().verified_only_select(sql); + assert_eq!(Some(Expr::Value(number("5"))), select.top); + assert!(select.percent); +} + #[test] fn parse_mssql_top() { - let sql = "SELECT TOP 5 [bar] FROM foo"; - let select = ms_and_generic().verified_query(sql); - assert_eq!(Some(Expr::Value(number("5"))), select.limit); + let sql = "SELECT TOP 5 bar, baz FROM foo"; + let _ = ms_and_generic().one_statement_parses_to(sql, "SELECT TOP (5) bar, baz FROM foo"); } fn ms() -> TestedDialects { From 2b89587bf6a032e9bea924ddcde2eeb190fe3e0b Mon Sep 17 00:00:00 2001 From: Alex Kyllo Date: Fri, 3 Jan 2020 23:27:03 -0800 Subject: [PATCH 3/7] fmt --- src/ast/query.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ast/query.rs b/src/ast/query.rs index 0e6e50ac6..f4ff5192f 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -134,11 +134,13 @@ impl fmt::Display for Select { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "SELECT{}", if self.distinct { " DISTINCT" } else { "" })?; if let Some(ref top) = self.top { - write!(f, - " TOP ({}){}{}", - top, - if self.percent { " PERCENT" } else { "" }, - if self.with_ties { " WITH TIES"} else { "" })?; + write!( + f, + " TOP ({}){}{}", + top, + if self.percent { " PERCENT" } else { "" }, + if self.with_ties { " WITH TIES" } else { "" } + )?; } write!(f, " {}", display_comma_separated(&self.projection))?; if !self.from.is_empty() { From ce2f0ca9bb4bc948feebdd91d1081a7fa56420bf Mon Sep 17 00:00:00 2001 From: Alex Kyllo Date: Fri, 3 Jan 2020 23:39:18 -0800 Subject: [PATCH 4/7] start refactoring TOP into its own struct --- src/ast/query.rs | 26 ++++++++++++++++++++++---- src/parser.rs | 32 ++++++++++++++++---------------- tests/sqlparser_mssql.rs | 16 ++++++++-------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/src/ast/query.rs b/src/ast/query.rs index f4ff5192f..63581d2e0 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -114,10 +114,8 @@ impl fmt::Display for SetOperator { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Select { pub distinct: bool, - /// top and percent are MSSQL only - pub top: Option, - pub percent: bool, - pub with_ties: bool, + /// MSSQL syntax: `TOP () [ PERCENT ] [ WITH TIES ]` + pub top: Option, /// projection expressions pub projection: Vec, /// FROM @@ -417,6 +415,26 @@ impl fmt::Display for Fetch { } } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Top { + /// SQL semantic equivalent of LIMIT but with same structure as FETCH. + pub with_ties: bool, + pub percent: bool, + pub quantity: Option, +} + +impl fmt::Display for Top { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let extension = if self.with_ties { " WITH TIES" } else { "" }; + if let Some(ref quantity) = self.quantity { + let percent = if self.percent { " PERCENT" } else { "" }; + write!(f, "TOP ({}) {}{}", quantity, percent, extension) + } else { + write!(f, "TOP{}", extension) + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Values(pub Vec>); diff --git a/src/parser.rs b/src/parser.rs index 09f1c03c5..06a995162 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1568,14 +1568,6 @@ impl Parser { None }; - let percent = self.parse_keyword("PERCENT"); - - let with_ties = self.parse_keywords(vec!["WITH", "TIES"]); - - if top.is_none() && percent { - return parser_err!("Cannot specify PERCENT without TOP in SELECT"); - } - let projection = self.parse_comma_separated(Parser::parse_select_item)?; // Note that for keywords to be properly handled here, they need to be @@ -1610,8 +1602,6 @@ impl Parser { Ok(Select { distinct, top, - percent, - with_ties, projection, from, selection, @@ -1960,14 +1950,24 @@ impl Parser { /// Parse a TOP clause, MSSQL equivalent of LIMIT, /// that follows after SELECT [DISTINCT]. - pub fn parse_top(&mut self) -> Result, ParserError> { - if self.consume_token(&Token::LParen) { - let expr = self.parse_expr()?; + pub fn parse_top(&mut self) -> Result { + let quantity = if self.consume_token(&Token::LParen) { + let quantity = self.parse_expr()?; self.expect_token(&Token::RParen)?; - Ok(Some(expr)) + Some(quantity) } else { - Ok(Some(Expr::Value(self.parse_number_value()?))) - } + Some(Expr::Value(self.parse_number_value()?)) + }; + + let percent = self.parse_keyword("PERCENT"); + + let with_ties = self.parse_keywords(vec!["WITH", "TIES"]); + + Ok(Top { + with_ties, + percent, + quantity, + }) } /// Parse a LIMIT clause diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index ee05a4810..d2cc1f1d2 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -72,32 +72,32 @@ fn parse_mssql_apply_join() { fn parse_mssql_top_paren() { let sql = "SELECT TOP (5) * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - assert_eq!(Some(Expr::Value(number("5"))), select.top); - assert!(!select.percent); + // assert_eq!(Some(Expr::Value(number("5"))), select.top); + // assert!(!select.percent); } #[test] fn parse_mssql_top_percent() { let sql = "SELECT TOP (5) PERCENT * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - assert_eq!(Some(Expr::Value(number("5"))), select.top); - assert!(select.percent); + // assert_eq!(Some(Expr::Value(number("5"))), select.top); + // assert!(select.percent); } #[test] fn parse_mssql_top_with_ties() { let sql = "SELECT TOP (5) WITH TIES * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - assert_eq!(Some(Expr::Value(number("5"))), select.top); - assert!(select.with_ties); + // assert_eq!(Some(Expr::Value(number("5"))), select.top); + //assert!(select.with_ties); } #[test] fn parse_mssql_top_percent_with_ties() { let sql = "SELECT TOP (5) PERCENT WITH TIES * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - assert_eq!(Some(Expr::Value(number("5"))), select.top); - assert!(select.percent); + // assert_eq!(Some(Expr::Value(number("5"))), select.top); + // assert!(select.percent); } #[test] From 069b954aa1379a67967a04e78c681c882ed039ff Mon Sep 17 00:00:00 2001 From: Alex Kyllo Date: Sat, 4 Jan 2020 09:01:42 -0800 Subject: [PATCH 5/7] refactor top/percent/with ties out of Select and into new nested child struct Top, similar to Fetch struct --- src/ast/mod.rs | 2 +- src/ast/query.rs | 10 ++-------- src/parser.rs | 2 +- tests/sqlparser_mssql.rs | 22 +++++++++++++--------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index eaf99b31b..2f723f012 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -27,7 +27,7 @@ pub use self::ddl::{ pub use self::operator::{BinaryOperator, UnaryOperator}; pub use self::query::{ Cte, Fetch, Join, JoinConstraint, JoinOperator, OrderByExpr, Query, Select, SelectItem, - SetExpr, SetOperator, TableAlias, TableFactor, TableWithJoins, Values, + SetExpr, SetOperator, TableAlias, TableFactor, TableWithJoins, Top, Values, }; pub use self::value::{DateTimeField, Value}; diff --git a/src/ast/query.rs b/src/ast/query.rs index 63581d2e0..a5eea141f 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -132,13 +132,7 @@ impl fmt::Display for Select { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "SELECT{}", if self.distinct { " DISTINCT" } else { "" })?; if let Some(ref top) = self.top { - write!( - f, - " TOP ({}){}{}", - top, - if self.percent { " PERCENT" } else { "" }, - if self.with_ties { " WITH TIES" } else { "" } - )?; + write!(f, " {}", top)?; } write!(f, " {}", display_comma_separated(&self.projection))?; if !self.from.is_empty() { @@ -428,7 +422,7 @@ impl fmt::Display for Top { let extension = if self.with_ties { " WITH TIES" } else { "" }; if let Some(ref quantity) = self.quantity { let percent = if self.percent { " PERCENT" } else { "" }; - write!(f, "TOP ({}) {}{}", quantity, percent, extension) + write!(f, "TOP ({}){}{}", quantity, percent, extension) } else { write!(f, "TOP{}", extension) } diff --git a/src/parser.rs b/src/parser.rs index 06a995162..85bb517fb 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1563,7 +1563,7 @@ impl Parser { } let top = if self.parse_keyword("TOP") { - self.parse_top()? + Some(self.parse_top()?) } else { None }; diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index d2cc1f1d2..2774d43ef 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -72,32 +72,36 @@ fn parse_mssql_apply_join() { fn parse_mssql_top_paren() { let sql = "SELECT TOP (5) * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - // assert_eq!(Some(Expr::Value(number("5"))), select.top); - // assert!(!select.percent); + let top = select.top.unwrap(); + assert_eq!(Some(Expr::Value(number("5"))), top.quantity); + assert!(!top.percent); } #[test] fn parse_mssql_top_percent() { let sql = "SELECT TOP (5) PERCENT * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - // assert_eq!(Some(Expr::Value(number("5"))), select.top); - // assert!(select.percent); + let top = select.top.unwrap(); + assert_eq!(Some(Expr::Value(number("5"))), top.quantity); + assert!(top.percent); } #[test] fn parse_mssql_top_with_ties() { let sql = "SELECT TOP (5) WITH TIES * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - // assert_eq!(Some(Expr::Value(number("5"))), select.top); - //assert!(select.with_ties); + let top = select.top.unwrap(); + assert_eq!(Some(Expr::Value(number("5"))), top.quantity); + assert!(top.with_ties); } #[test] fn parse_mssql_top_percent_with_ties() { - let sql = "SELECT TOP (5) PERCENT WITH TIES * FROM foo"; + let sql = "SELECT TOP (10) PERCENT WITH TIES * FROM foo"; let select = ms_and_generic().verified_only_select(sql); - // assert_eq!(Some(Expr::Value(number("5"))), select.top); - // assert!(select.percent); + let top = select.top.unwrap(); + assert_eq!(Some(Expr::Value(number("10"))), top.quantity); + assert!(top.percent); } #[test] From 71b3a5811018044ba756fa720099cf236a703439 Mon Sep 17 00:00:00 2001 From: Alex Kyllo Date: Sat, 4 Jan 2020 09:58:57 -0800 Subject: [PATCH 6/7] remove double #[must_use] to fix cargo clippy error --- src/parser.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/parser.rs b/src/parser.rs index 85bb517fb..c9e32ed3b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -783,7 +783,6 @@ impl Parser { } /// Bail out if the current token is not one of the expected keywords, or consume it if it is - #[must_use] pub fn expect_one_of_keywords( &mut self, keywords: &[&'static str], From 5e5a177329d257c532c8d4962a4fc5e44386cc12 Mon Sep 17 00:00:00 2001 From: Alex Kyllo Date: Sat, 4 Jan 2020 10:02:02 -0800 Subject: [PATCH 7/7] remove unnecessary .clone() call in mysql test to fix clippy::redundant-clone error --- tests/sqlparser_mysql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index ce9d0053b..cc6433322 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -77,7 +77,7 @@ fn parse_show_columns() { Statement::ShowColumns { extended: false, full: false, - table_name: table_name.clone(), + table_name: table_name, filter: Some(ShowStatementFilter::Where( mysql_and_generic().verified_expr("1 = 2") )),