From 12a29ec82df399eadf83135bfe5763730cc7c386 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 27 Apr 2022 15:13:09 +0300 Subject: [PATCH 1/6] Fix redshift specific --- src/dialect/postgresql.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 0c2eb99f0..87d9fd9d4 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -20,7 +20,7 @@ impl Dialect for PostgreSqlDialect { // See https://www.postgresql.org/docs/11/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS // We don't yet support identifiers beginning with "letters with // diacritical marks and non-Latin letters" - ('a'..='z').contains(&ch) || ('A'..='Z').contains(&ch) || ch == '_' + ('a'..='z').contains(&ch) || ('A'..='Z').contains(&ch) || ch == '_' || ch == '#' } fn is_identifier_part(&self, ch: char) -> bool { @@ -29,5 +29,6 @@ impl Dialect for PostgreSqlDialect { || ('0'..='9').contains(&ch) || ch == '$' || ch == '_' + || ch == '#' } } From eb2ff88916973ca21879699e69119ab68db97e5b Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 27 Apr 2022 15:59:49 +0300 Subject: [PATCH 2/6] Temp fix --- src/tokenizer.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 566deacec..2d6b3b8d6 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -422,7 +422,24 @@ impl<'a> Tokenizer<'a> { s += s2.as_str(); return Ok(Some(Token::Number(s, false))); } - Ok(Some(Token::make_word(&s, None))) + if s == "#" { + match chars.peek() { + Some('>') => { + chars.next(); + match chars.peek() { + Some('>') => { + chars.next(); + return Ok(Some(Token::HashLongArrow)) + } + _ => Ok(Some(Token::HashArrow)), + } + } + _ => Ok(Some(Token::Sharp)), + } + } else { + Ok(Some(Token::make_word(&s, None))) + } + } // string '\'' => { From cbce74fdcc8cc85c8c7933f2d1ac7726a7d4b573 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 27 Apr 2022 16:03:57 +0300 Subject: [PATCH 3/6] Extract sharp to a function --- src/tokenizer.rs | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2d6b3b8d6..dbec49d81 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -361,6 +361,22 @@ impl<'a> Tokenizer<'a> { Ok(tokens) } + fn consume_sharp(&self, chars: &mut Peekable>) -> Result, TokenizerError> { + match chars.peek() { + Some('>') => { + chars.next(); + match chars.peek() { + Some('>') => { + chars.next(); + return Ok(Some(Token::HashLongArrow)) + } + _ => return Ok(Some(Token::HashArrow)), + } + } + _ => return Ok(Some(Token::Sharp)), + } + } + /// Get the next token or return None fn next_token(&self, chars: &mut Peekable>) -> Result, TokenizerError> { //println!("next_token: {:?}", chars.peek()); @@ -423,19 +439,7 @@ impl<'a> Tokenizer<'a> { return Ok(Some(Token::Number(s, false))); } if s == "#" { - match chars.peek() { - Some('>') => { - chars.next(); - match chars.peek() { - Some('>') => { - chars.next(); - return Ok(Some(Token::HashLongArrow)) - } - _ => Ok(Some(Token::HashArrow)), - } - } - _ => Ok(Some(Token::Sharp)), - } + self.consume_sharp(chars) } else { Ok(Some(Token::make_word(&s, None))) } @@ -641,19 +645,7 @@ impl<'a> Tokenizer<'a> { } '#' => { chars.next(); - match chars.peek() { - Some('>') => { - chars.next(); - match chars.peek() { - Some('>') => { - chars.next(); - Ok(Some(Token::HashLongArrow)) - } - _ => Ok(Some(Token::HashArrow)), - } - } - _ => Ok(Some(Token::Sharp)), - } + self.consume_sharp(chars) } '@' => self.consume_and_return(chars, Token::AtSign), '?' => self.consume_and_return(chars, Token::Placeholder(String::from("?"))), From 9fccab9f448b8a1122a7bbec78af3a3ef7a68348 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 27 Apr 2022 16:07:04 +0300 Subject: [PATCH 4/6] Fix lint --- src/tokenizer.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index dbec49d81..441876db0 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -361,14 +361,17 @@ impl<'a> Tokenizer<'a> { Ok(tokens) } - fn consume_sharp(&self, chars: &mut Peekable>) -> Result, TokenizerError> { + fn consume_sharp( + &self, + chars: &mut Peekable>, + ) -> Result, TokenizerError> { match chars.peek() { Some('>') => { chars.next(); match chars.peek() { Some('>') => { chars.next(); - return Ok(Some(Token::HashLongArrow)) + return Ok(Some(Token::HashLongArrow)); } _ => return Ok(Some(Token::HashArrow)), } @@ -443,7 +446,6 @@ impl<'a> Tokenizer<'a> { } else { Ok(Some(Token::make_word(&s, None))) } - } // string '\'' => { From 96fda520fa60a815f94b5384f3755321197c7db3 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 27 Apr 2022 16:10:33 +0300 Subject: [PATCH 5/6] Fix lint --- src/tokenizer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 441876db0..525652ba1 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -371,12 +371,12 @@ impl<'a> Tokenizer<'a> { match chars.peek() { Some('>') => { chars.next(); - return Ok(Some(Token::HashLongArrow)); + Ok(Some(Token::HashLongArrow)) } - _ => return Ok(Some(Token::HashArrow)), + _ => Ok(Some(Token::HashArrow)), } } - _ => return Ok(Some(Token::Sharp)), + _ => Ok(Some(Token::Sharp)), } } From d4f75986406afc4dbdfb115099b4f3a400596126 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 27 Apr 2022 16:16:31 +0300 Subject: [PATCH 6/6] Add test --- tests/sqlparser_postgres.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index eb42edc8d..c60e06c06 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -1373,3 +1373,13 @@ fn pg_and_generic() -> TestedDialects { dialects: vec![Box::new(PostgreSqlDialect {}), Box::new(GenericDialect {})], } } + +#[test] +fn test_sharp() { + let sql = "SELECT #_of_values"; + let select = pg().verified_only_select(sql); + assert_eq!( + SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("#_of_values"))), + select.projection[0] + ); +}