diff --git a/crates/pg_lexer/src/lib.rs b/crates/pg_lexer/src/lib.rs index df24f8d85..0bf933e97 100644 --- a/crates/pg_lexer/src/lib.rs +++ b/crates/pg_lexer/src/lib.rs @@ -89,7 +89,7 @@ fn whitespace_tokens(input: &str) -> VecDeque { } else if let Some(tab) = cap.name("tab") { tokens.push_back(Token { token_type: TokenType::Whitespace, - kind: SyntaxKind::Newline, + kind: SyntaxKind::Tab, text: tab.as_str().to_string(), span: TextRange::new( TextSize::from(u32::try_from(tab.start()).unwrap()), @@ -177,9 +177,30 @@ pub fn lex(text: &str) -> Vec { mod tests { use super::*; + #[test] + fn test_tab_tokens() { + let input = "select\t1"; + let tokens = lex(input); + assert_eq!(tokens[1].kind, SyntaxKind::Tab); + } + + #[test] + fn test_newline_tokens() { + let input = "select\n1"; + let tokens = lex(input); + assert_eq!(tokens[1].kind, SyntaxKind::Newline); + } + + #[test] + fn test_whitespace_tokens() { + let input = "select 1"; + let tokens = lex(input); + assert_eq!(tokens[1].kind, SyntaxKind::Whitespace); + } + #[test] fn test_lexer() { - let input = "select 1; \n -- some comment \n select 2"; + let input = "select 1; \n -- some comment \n select 2\t"; let tokens = lex(input); let mut tokens_iter = tokens.iter(); @@ -227,5 +248,8 @@ mod tests { let token = tokens_iter.next().unwrap(); assert_eq!(token.kind, SyntaxKind::Iconst); assert_eq!(token.text, "2"); + + let token = tokens_iter.next().unwrap(); + assert_eq!(token.kind, SyntaxKind::Tab); } }