Merge pull request #7071 from andylokandy/dialect

feat(parser): add mysql dialect
datafuselabs · Aug 10, 2022 · a7f2c58 · a7f2c58 · vercel · Aug 10, 2022
2 parents dc289e0 + d8c99fb
commit a7f2c58
Show file tree

Hide file tree

Showing 21 changed files with 102 additions and 45 deletions.
diff --git a/common/ast/src/error.rs b/common/ast/src/error.rs
@@ -88,7 +88,7 @@ impl<'a> nom::error::ParseError<Input<'a>> for Error<'a> {
             span: i[0].clone(),
             errors: vec![],
             contexts: vec![],
-            backtrace: i.1,
+            backtrace: i.2,
         }
     }
 
@@ -122,7 +122,7 @@ impl<'a> nom::error::ContextError<Input<'a>> for Error<'a> {
 
 impl<'a> Error<'a> {
     pub fn from_error_kind(input: Input<'a>, kind: ErrorKind) -> Self {
-        let mut inner = input.1.inner.borrow_mut();
+        let mut inner = input.2.inner.borrow_mut();
         if let Some(ref mut inner) = *inner {
             match input.0[0].span.start.cmp(&inner.span.span.start) {
                 Ordering::Equal => {
@@ -147,7 +147,7 @@ impl<'a> Error<'a> {
             span: input.0[0].clone(),
             errors: vec![kind],
             contexts: vec![],
-            backtrace: input.1,
+            backtrace: input.2,
         }
     }
 }

diff --git a/common/ast/src/input.rs b/common/ast/src/input.rs
@@ -23,7 +23,7 @@ use crate::Backtrace;
 /// Input tokens slice with a backtrace that records all errors including
 /// the optional branch.
 #[derive(Debug, Clone, Copy)]
-pub struct Input<'a>(pub &'a [Token<'a>], pub &'a Backtrace<'a>);
+pub struct Input<'a>(pub &'a [Token<'a>], pub Dialect, pub &'a Backtrace<'a>);
 
 impl<'a> std::ops::Deref for Input<'a> {
     type Target = [Token<'a>];
@@ -50,19 +50,19 @@ impl<'a> nom::Offset for Input<'a> {
 
 impl<'a> nom::Slice<Range<usize>> for Input<'a> {
     fn slice(&self, range: Range<usize>) -> Self {
-        Input(&self.0[range], self.1)
+        Input(&self.0[range], self.1, self.2)
     }
 }
 
 impl<'a> nom::Slice<RangeTo<usize>> for Input<'a> {
     fn slice(&self, range: RangeTo<usize>) -> Self {
-        Input(&self.0[range], self.1)
+        Input(&self.0[range], self.1, self.2)
     }
 }
 
 impl<'a> nom::Slice<RangeFrom<usize>> for Input<'a> {
     fn slice(&self, range: RangeFrom<usize>) -> Self {
-        Input(&self.0[range], self.1)
+        Input(&self.0[range], self.1, self.2)
     }
 }
 
@@ -77,3 +77,27 @@ pub struct WithSpan<'a, T> {
     pub(crate) span: Input<'a>,
     pub(crate) elem: T,
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum Dialect {
+    #[default]
+    PostgreSQL,
+    MySQL,
+}
+
+impl Dialect {
+    pub fn is_ident_quote(&self, c: char) -> bool {
+        match self {
+            Dialect::MySQL => c == '`',
+            // TODO: remove '`' quote support once mysql handler correctly set mysql dialect.
+            Dialect::PostgreSQL => c == '"' || c == '`',
+        }
+    }
+
+    pub fn is_string_quote(&self, c: char) -> bool {
+        match self {
+            Dialect::MySQL => c == '\'' || c == '"',
+            Dialect::PostgreSQL => c == '\'',
+        }
+    }
+}
diff --git a/common/ast/src/lib.rs b/common/ast/src/lib.rs
@@ -29,6 +29,7 @@ pub use visitors::Visitor;
 pub use visitors::VisitorMut;
 
 mod input;
+pub use input::Dialect;
 pub use input::Input;
 
 mod util;

diff --git a/common/ast/src/parser/expr.rs b/common/ast/src/parser/expr.rs
@@ -1080,7 +1080,13 @@ pub fn literal_string(i: Input) -> IResult<String> {
             QuotedString
         },
         |token| {
-            if token.text().starts_with('\'') {
+            if token
+                .text()
+                .chars()
+                .next()
+                .filter(|c| i.1.is_string_quote(*c))
+                .is_some()
+            {
                 let str = &token.text()[1..token.text().len() - 1];
                 let unescaped =
                     unescape(str, '\'').ok_or(ErrorKind::Other("invalid escape or unicode"))?;

diff --git a/common/ast/src/parser/mod.rs b/common/ast/src/parser/mod.rs
@@ -24,6 +24,7 @@ use common_exception::Result;
 use self::expr::subexpr;
 use crate::ast::Expr;
 use crate::ast::Statement;
+use crate::input::Dialect;
 use crate::input::Input;
 use crate::parser::statement::statement;
 use crate::parser::token::Token;
@@ -40,9 +41,10 @@ pub fn tokenize_sql(sql: &str) -> Result<Vec<Token>> {
 /// Parse a SQL string into `Statement`s.
 pub fn parse_sql<'a>(
     sql_tokens: &'a [Token<'a>],
+    dialect: Dialect,
     backtrace: &'a Backtrace<'a>,
 ) -> Result<(Statement<'a>, Option<String>)> {
-    match statement(Input(sql_tokens, backtrace)) {
+    match statement(Input(sql_tokens, dialect, backtrace)) {
         Ok((rest, stmts)) if rest[0].kind == TokenKind::EOI => Ok((stmts.stmt, stmts.format)),
         Ok((rest, _)) => Err(ErrorCode::SyntaxException(
             rest[0].display_error("unable to parse rest of the sql".to_string()),
@@ -57,9 +59,10 @@ pub fn parse_sql<'a>(
 /// Parse udf function into Expr
 pub fn parse_expr<'a>(
     sql_tokens: &'a [Token<'a>],
+    dialect: Dialect,
     backtrace: &'a Backtrace<'a>,
 ) -> Result<Expr<'a>> {
-    match expr::expr(Input(sql_tokens, backtrace)) {
+    match expr::expr(Input(sql_tokens, dialect, backtrace)) {
         Ok((rest, expr)) if rest[0].kind == TokenKind::EOI => Ok(expr),
         Ok((rest, _)) => Err(ErrorCode::SyntaxException(
             rest[0].display_error("unable to parse rest of the sql".to_string()),
@@ -73,10 +76,11 @@ pub fn parse_expr<'a>(
 
 pub fn parse_comma_separated_exprs<'a>(
     sql_tokens: &'a [Token<'a>],
+    dialect: Dialect,
     backtrace: &'a Backtrace<'a>,
 ) -> Result<Vec<Expr<'a>>> {
     let mut comma_separated_exprs_parser = comma_separated_list0(subexpr(0));
-    match comma_separated_exprs_parser(Input(sql_tokens, backtrace)) {
+    match comma_separated_exprs_parser(Input(sql_tokens, dialect, backtrace)) {
         Ok((_rest, exprs)) => Ok(exprs),
         Err(nom::Err::Error(err) | nom::Err::Failure(err)) => {
             Err(ErrorCode::SyntaxException(err.display_error(())))

diff --git a/common/ast/src/util.rs b/common/ast/src/util.rs
@@ -92,17 +92,23 @@ fn non_reserved_identifier(
             ),
             move |i| {
                 match_token(QuotedString)(i).and_then(|(i2, token)| {
-                    if token.text().starts_with('\'') {
-                        Err(nom::Err::Error(Error::from_error_kind(
-                            i,
-                            ErrorKind::ExpectToken(Ident),
-                        )))
-                    } else {
+                    if token
+                        .text()
+                        .chars()
+                        .next()
+                        .filter(|c| i.1.is_ident_quote(*c))
+                        .is_some()
+                    {
                         Ok((i2, Identifier {
                             span: token.clone(),
                             name: token.text()[1..token.text().len() - 1].to_string(),
                             quote: Some(token.text().chars().next().unwrap()),
                         }))
+                    } else {
+                        Err(nom::Err::Error(Error::from_error_kind(
+                            i,
+                            ErrorKind::ExpectToken(Ident),
+                        )))
                     }
                 })
             },
@@ -333,7 +339,7 @@ where
         .map_err(nom::Err::Error)?;
     if let Some(elem) = iter.peek() {
         // Rollback parsing footprint on unused expr elements.
-        input.1.clear();
+        input.2.clear();
         Ok((input.slice(input.offset(&elem.span)..), expr))
     } else {
         Ok((rest, expr))

diff --git a/common/ast/tests/it/parser.rs b/common/ast/tests/it/parser.rs
@@ -21,6 +21,7 @@ use common_ast::parser::token::*;
 use common_ast::parser::tokenize_sql;
 use common_ast::rule;
 use common_ast::Backtrace;
+use common_ast::Dialect;
 use common_ast::DisplayError;
 use common_ast::Input;
 use common_exception::Result;
@@ -33,7 +34,7 @@ macro_rules! run_parser {
         let backtrace = Backtrace::new();
         let parser = $parser;
         let mut parser = rule! { #parser ~ &EOI };
-        match parser.parse(Input(&tokens, &backtrace)) {
+        match parser.parse(Input(&tokens, Dialect::PostgreSQL, &backtrace)) {
             Ok((i, (output, _))) => {
                 assert_eq!(i[0].kind, TokenKind::EOI);
                 writeln!($file, "---------- Input ----------").unwrap();
@@ -109,7 +110,7 @@ fn test_statement() {
         r#"select * from t4;"#,
         r#"select * from aa.bb;"#,
         r#"select * from a, b, c;"#,
-        r#"select * from a, b, c order by `db`.`a`.`c1`;"#,
+        r#"select * from a, b, c order by "db"."a"."c1";"#,
         r#"select * from a join b on a.a = b.a;"#,
         r#"select * from a left outer join b on a.a = b.a;"#,
         r#"select * from a right outer join b on a.a = b.a;"#,
@@ -269,7 +270,7 @@ fn test_statement() {
     for case in cases {
         let tokens = tokenize_sql(case).unwrap();
         let backtrace = Backtrace::new();
-        let (stmt, fmt) = parse_sql(&tokens, &backtrace).unwrap();
+        let (stmt, fmt) = parse_sql(&tokens, Dialect::PostgreSQL, &backtrace).unwrap();
         writeln!(file, "---------- Input ----------").unwrap();
         writeln!(file, "{}", case).unwrap();
         writeln!(file, "---------- Output ---------").unwrap();
@@ -325,7 +326,7 @@ fn test_statement_error() {
     for case in cases {
         let tokens = tokenize_sql(case).unwrap();
         let backtrace = Backtrace::new();
-        let err = parse_sql(&tokens, &backtrace).unwrap_err();
+        let err = parse_sql(&tokens, Dialect::PostgreSQL, &backtrace).unwrap_err();
         writeln!(file, "---------- Input ----------").unwrap();
         writeln!(file, "{}", case).unwrap();
         writeln!(file, "---------- Output ---------").unwrap();
@@ -426,7 +427,7 @@ fn test_expr() {
         r#"1 - -(- - -1)"#,
         r#"1 + a * c.d"#,
         r#"number % 2"#,
-        r#"`t`:k1.k2"#,
+        r#""t":k1.k2"#,
         r#"col1 not between 1 and 2"#,
         r#"sum(col1)"#,
         r#""random"()"#,

diff --git a/common/ast/tests/it/testdata/expr.txt b/common/ast/tests/it/testdata/expr.txt
@@ -1116,9 +1116,9 @@ BinaryOp {
 
 
 ---------- Input ----------
-`t`:k1.k2
+"t":k1.k2
 ---------- Output ---------
-`t`:k1.k2
+"t":k1.k2
 ---------- AST ------------
 MapAccess {
     span: [
@@ -1139,7 +1139,7 @@ MapAccess {
             column: Identifier {
                 name: "t",
                 quote: Some(
-                    '`',
+                    '"',
                 ),
                 span: QuotedString(0..3),
             },

diff --git a/common/ast/tests/it/testdata/statement.txt b/common/ast/tests/it/testdata/statement.txt
@@ -2027,9 +2027,9 @@ Query(
 
 
 ---------- Input ----------
-select * from a, b, c order by `db`.`a`.`c1`;
+select * from a, b, c order by "db"."a"."c1";
 ---------- Output ---------
-SELECT * FROM a, b, c ORDER BY `db`.`a`.`c1`
+SELECT * FROM a, b, c ORDER BY "db"."a"."c1"
 ---------- AST ------------
 Query(
     Query {
@@ -2134,7 +2134,7 @@ Query(
                         Identifier {
                             name: "db",
                             quote: Some(
-                                '`',
+                                '"',
                             ),
                             span: QuotedString(31..35),
                         },
@@ -2143,15 +2143,15 @@ Query(
                         Identifier {
                             name: "a",
                             quote: Some(
-                                '`',
+                                '"',
                             ),
                             span: QuotedString(36..39),
                         },
                     ),
                     column: Identifier {
                         name: "c1",
                         quote: Some(
-                            '`',
+                            '"',
                         ),
                         span: QuotedString(40..44),
                     },

diff --git a/common/expression/tests/it/parser.rs b/common/expression/tests/it/parser.rs
@@ -17,6 +17,7 @@ use common_ast::parser::parse_expr;
 use common_ast::parser::token::Token;
 use common_ast::parser::tokenize_sql;
 use common_ast::Backtrace;
+use common_ast::Dialect;
 use common_expression::types::DataType;
 use common_expression::Literal;
 use common_expression::RawExpr;
@@ -25,7 +26,7 @@ use common_expression::Span;
 pub fn parse_raw_expr(text: &str, columns: &[(&str, DataType)]) -> RawExpr {
     let backtrace = Backtrace::new();
     let tokens = tokenize_sql(text).unwrap();
-    let expr = parse_expr(&tokens, &backtrace).unwrap();
+    let expr = parse_expr(&tokens, Dialect::PostgreSQL, &backtrace).unwrap();
     transform_expr(expr, columns)
 }
 

diff --git a/common/functions-v2/tests/it/scalars/parser.rs b/common/functions-v2/tests/it/scalars/parser.rs
@@ -17,6 +17,7 @@ use common_ast::parser::parse_expr;
 use common_ast::parser::token::Token;
 use common_ast::parser::tokenize_sql;
 use common_ast::Backtrace;
+use common_ast::Dialect;
 use common_expression::types::DataType;
 use common_expression::Literal;
 use common_expression::RawExpr;
@@ -25,7 +26,7 @@ use common_expression::Span;
 pub fn parse_raw_expr(text: &str, columns: &[(&str, DataType)]) -> RawExpr {
     let backtrace = Backtrace::new();
     let tokens = tokenize_sql(text).unwrap();
-    let expr = parse_expr(&tokens, &backtrace).unwrap();
+    let expr = parse_expr(&tokens, Dialect::PostgreSQL, &backtrace).unwrap();
     transform_expr(expr, columns)
 }
 

diff --git a/query/src/sql/planner/binder/copy.rs b/query/src/sql/planner/binder/copy.rs
@@ -21,6 +21,7 @@ use common_ast::ast::Statement;
 use common_ast::parser::parse_sql;
 use common_ast::parser::tokenize_sql;
 use common_ast::Backtrace;
+use common_ast::Dialect;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use common_meta_types::UserStageInfo;
@@ -325,7 +326,7 @@ impl<'a> Binder {
             format!("SELECT * FROM {src_catalog_name}.{src_database_name}.{src_table_name}");
         let tokens = tokenize_sql(&subquery)?;
         let backtrace = Backtrace::new();
-        let sub_stmt_msg = parse_sql(&tokens, &backtrace)?;
+        let sub_stmt_msg = parse_sql(&tokens, Dialect::PostgreSQL, &backtrace)?;
         let sub_stmt = sub_stmt_msg.0;
         let query = match &sub_stmt {
             Statement::Query(query) => {
@@ -370,7 +371,7 @@ impl<'a> Binder {
             format!("SELECT * FROM {src_catalog_name}.{src_database_name}.{src_table_name}");
         let tokens = tokenize_sql(&subquery)?;
         let backtrace = Backtrace::new();
-        let sub_stmt_msg = parse_sql(&tokens, &backtrace)?;
+        let sub_stmt_msg = parse_sql(&tokens, Dialect::PostgreSQL, &backtrace)?;
         let sub_stmt = sub_stmt_msg.0;
         let query = match &sub_stmt {
             Statement::Query(query) => {

diff --git a/query/src/sql/planner/binder/ddl/table.rs b/query/src/sql/planner/binder/ddl/table.rs
@@ -23,6 +23,7 @@ use common_ast::parser::parse_sql;
 use common_ast::parser::tokenize_sql;
 use common_ast::walk_expr_mut;
 use common_ast::Backtrace;
+use common_ast::Dialect;
 use common_datavalues::DataField;
 use common_datavalues::DataSchemaRef;
 use common_datavalues::DataSchemaRefExt;
@@ -304,7 +305,7 @@ impl<'a> Binder {
         };
         let tokens = tokenize_sql(query.as_str())?;
         let backtrace = Backtrace::new();
-        let (stmt, _) = parse_sql(&tokens, &backtrace)?;
+        let (stmt, _) = parse_sql(&tokens, Dialect::PostgreSQL, &backtrace)?;
         self.bind_statement(bind_context, &stmt).await
     }