From b24f55fc576c225a7fe8a242ca0e6c9c9b564f18 Mon Sep 17 00:00:00 2001
From: "Heres, Daniel" <danielheres@gmail.com>
Date: Sun, 7 Feb 2021 15:34:14 +0100
Subject: [PATCH 1/4] Parse floats without leading number

---
 src/tokenizer.rs | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index fd33f9589..7a1bf0a62 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -395,16 +395,25 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 // numbers
-                '0'..='9' => {
-                    // TODO: https://jakewheat.github.io/sql-overview/sql-2011-foundation-grammar.html#unsigned-numeric-literal
-                    let s = peeking_take_while(chars, |ch| matches!(ch, '0'..='9' | '.'));
+                '0'..='9' | '.' => {
+                    let mut s = peeking_take_while(chars, |ch| matches!(ch, '0'..='9'));
+                    if let Some('.') = chars.peek() {
+                        s.push('.');
+                        chars.next();
+                    }
+                    s += &peeking_take_while(chars, |ch| matches!(ch, '0'..='9'));
+
                     let long = if chars.peek() == Some(&'L') {
                         chars.next();
                         true
                     } else {
                         false
                     };
-                    Ok(Some(Token::Number(s, long)))
+                    if s == "." {
+                        Ok(Some(Token::Period))
+                    } else {
+                        Ok(Some(Token::Number(s, long)))
+                    }
                 }
                 // punctuation
                 '(' => self.consume_and_return(chars, Token::LParen),
@@ -470,7 +479,6 @@ impl<'a> Tokenizer<'a> {
                         _ => Ok(Some(Token::Eq)),
                     }
                 }
-                '.' => self.consume_and_return(chars, Token::Period),
                 '!' => {
                     chars.next(); // consume
                     match chars.peek() {
@@ -667,6 +675,22 @@ mod tests {
         compare(expected, tokens);
     }
 
+    #[test]
+    fn tokenize_select_float() {
+        let sql = String::from("SELECT .1");
+        let dialect = GenericDialect {};
+        let mut tokenizer = Tokenizer::new(&dialect, &sql);
+        let tokens = tokenizer.tokenize().unwrap();
+
+        let expected = vec![
+            Token::make_keyword("SELECT"),
+            Token::Whitespace(Whitespace::Space),
+            Token::Number(String::from(".1"), false),
+        ];
+
+        compare(expected, tokens);
+    }
+
     #[test]
     fn tokenize_scalar_function() {
         let sql = String::from("SELECT sqrt(1)");

From f09f302517866a58003465d28bf7402567e628fd Mon Sep 17 00:00:00 2001
From: "Heres, Daniel" <danielheres@gmail.com>
Date: Sun, 7 Feb 2021 15:36:57 +0100
Subject: [PATCH 2/4] Move period token test

---
 src/tokenizer.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 7a1bf0a62..1df1db758 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -403,17 +403,17 @@ impl<'a> Tokenizer<'a> {
                     }
                     s += &peeking_take_while(chars, |ch| matches!(ch, '0'..='9'));
 
+                    if s == "." {
+                        return Ok(Some(Token::Period));
+                    }
+
                     let long = if chars.peek() == Some(&'L') {
                         chars.next();
                         true
                     } else {
                         false
                     };
-                    if s == "." {
-                        Ok(Some(Token::Period))
-                    } else {
-                        Ok(Some(Token::Number(s, long)))
-                    }
+                    Ok(Some(Token::Number(s, long)))
                 }
                 // punctuation
                 '(' => self.consume_and_return(chars, Token::LParen),

From 40c87379118aa4115f078ee0edd1c871732cf98b Mon Sep 17 00:00:00 2001
From: "Heres, Daniel" <danielheres@gmail.com>
Date: Sun, 7 Feb 2021 15:38:03 +0100
Subject: [PATCH 3/4] Comments

---
 src/tokenizer.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 1df1db758..d82810528 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -394,15 +394,17 @@ impl<'a> Tokenizer<'a> {
                         )
                     }
                 }
-                // numbers
+                // numbers and period
                 '0'..='9' | '.' => {
                     let mut s = peeking_take_while(chars, |ch| matches!(ch, '0'..='9'));
+                    // match one period
                     if let Some('.') = chars.peek() {
                         s.push('.');
                         chars.next();
                     }
                     s += &peeking_take_while(chars, |ch| matches!(ch, '0'..='9'));
 
+                    // No number -> Token::Period
                     if s == "." {
                         return Ok(Some(Token::Period));
                     }

From c2c83c1c7d803f6f5daeda2f84234cd3d0c8ce4f Mon Sep 17 00:00:00 2001
From: "Heres, Daniel" <danielheres@gmail.com>
Date: Sun, 7 Feb 2021 15:38:47 +0100
Subject: [PATCH 4/4] Enable test

---
 tests/sqlparser_regression.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/sqlparser_regression.rs b/tests/sqlparser_regression.rs
index bbf1b2977..d98808a89 100644
--- a/tests/sqlparser_regression.rs
+++ b/tests/sqlparser_regression.rs
@@ -27,8 +27,8 @@ macro_rules! tpch_tests {
             let dialect = GenericDialect {};
 
             let res = Parser::parse_sql(&dialect, QUERIES[$value -1]);
-            // Ignore 6.sql and 22.sql
-            if $value != 6 && $value != 22 {
+            // Ignore 22.sql
+            if $value != 22 {
                 assert!(res.is_ok());
             }
         }