diff --git a/src/interpreter/tokenizer/mod.rs b/src/interpreter/tokenizer/mod.rs index 4b6ae9c..2da9292 100644 --- a/src/interpreter/tokenizer/mod.rs +++ b/src/interpreter/tokenizer/mod.rs @@ -231,4 +231,109 @@ mod tests { ]; assert_eq!(expected, result); } + + #[test] + fn tokenizer_parses_comments() { + let result = tokenize("SELECT * FROM users WHERE name = 'Fletcher'; -- This is a comment"); + let expected = vec![ + token(TokenTypes::Select, "SELECT", 0, 1), + token(TokenTypes::Asterisk, "*", 7, 1), + token(TokenTypes::From, "FROM", 9, 1), + token(TokenTypes::Identifier, "users", 14, 1), + token(TokenTypes::Where, "WHERE", 20, 1), + token(TokenTypes::Identifier, "name", 26, 1), + token(TokenTypes::Equals, "=", 31, 1), + token(TokenTypes::String, "Fletcher", 33, 1), + token(TokenTypes::SemiColon, ";", 43, 1), + token(TokenTypes::EOF, "", 0, 0), + ]; + assert_eq!(expected, result); + } + + #[test] + fn tokenizer_parses_comments_with_newlines() { + let result = tokenize(" + SELECT * FROM users; -- This is a comment + SELECT * FROM users;"); + let expected = vec![ + token(TokenTypes::Select, "SELECT", 8, 2), + token(TokenTypes::Asterisk, "*", 15, 2), + token(TokenTypes::From, "FROM", 17, 2), + token(TokenTypes::Identifier, "users", 22, 2), + token(TokenTypes::SemiColon, ";", 27, 2), + token(TokenTypes::Select, "SELECT", 8, 3), + token(TokenTypes::Asterisk, "*", 15, 3), + token(TokenTypes::From, "FROM", 17, 3), + token(TokenTypes::Identifier, "users", 22, 3), + token(TokenTypes::SemiColon, ";", 27, 3), + token(TokenTypes::EOF, "", 0, 0), + ]; + assert_eq!(expected, result); + } + + #[test] + fn tokenizer_parses_comments_with_asterisks() { + let result = tokenize("SELECT /* This is a comment */ SELECT "); + let expected = vec![ + token(TokenTypes::Select, "SELECT", 0, 1), + token(TokenTypes::Select, "SELECT", 32, 1), + token(TokenTypes::EOF, "", 0, 0), + ]; + assert_eq!(expected, result); + } + + #[test] + fn tokenizer_parses_comments_with_asterisks_and_newlines() { + let result = tokenize(" + SELECT + /* This is a comment */ + SELECT + "); + let expected = vec![ + token(TokenTypes::Select, "SELECT", 8, 2), + token(TokenTypes::Select, "SELECT", 8, 4), + token(TokenTypes::EOF, "", 0, 0), + ]; + assert_eq!(expected, result); + } + + #[test] + fn tokenizer_parses_comment_block_spans_multiple_lines() { + let result = tokenize(" + SELECT + /* + This is a comment + that is multi line + */ + SELECT + "); + let expected = vec![ + token(TokenTypes::Select, "SELECT", 8, 2), + token(TokenTypes::Select, "SELECT", 8, 7), + token(TokenTypes::EOF, "", 0, 0), + ]; + assert_eq!(expected, result); + } + + #[test] + fn tokenizer_raises_error_when_comment_block_is_not_closed() { + let result = tokenize("SELECT /* This is a comment"); + let expected = vec![ + token(TokenTypes::Select, "SELECT", 0, 1), + token(TokenTypes::Error, " This is a comment", 9, 1), + token(TokenTypes::EOF, "", 0, 0), + ]; + assert_eq!(expected, result); + } + + #[test] + fn tokenizer_raises_error_when_comment_block_is_not_closed_with_slash() { + let result = tokenize("SELECT /* This is a comment * SELECT"); + let expected = vec![ + token(TokenTypes::Select, "SELECT", 0, 1), + token(TokenTypes::Error, " This is a comment * SELECT", 9, 1), + token(TokenTypes::EOF, "", 0, 0), + ]; + assert_eq!(expected, result); + } } \ No newline at end of file diff --git a/src/interpreter/tokenizer/scanner.rs b/src/interpreter/tokenizer/scanner.rs index 8372e6e..868a975 100644 --- a/src/interpreter/tokenizer/scanner.rs +++ b/src/interpreter/tokenizer/scanner.rs @@ -223,6 +223,33 @@ impl<'a> Scanner<'a> { } } + fn read_block_comment(&mut self, start: usize) -> Option> { + self.advance(); + self.advance(); + + while self.current < self.input.len() { + if self.current_char() == '*' && self.current + 1 < self.input.len() && self.peek_char() == '/' { + self.advance(); + self.advance(); + return self.next_token(); + } + + if self.current_char() == '\n' { + self.line_num += 1; + self.col_num = self.current + 1; + } + + self.advance(); + } + + Some(Token { + token_type: TokenTypes::Error, + value: &self.input[start + 2..self.current], + col_num: start - self.col_num + 2, + line_num: self.line_num, + }) + } + pub fn next_token(&mut self) -> Option> { while self.handle_skips() {} @@ -272,11 +299,29 @@ impl<'a> Scanner<'a> { self.advance(); let token_type = self.read_digit(); Some(self.build_token(start, token_type)) - } else { + } else if self.peek_char() == '-' { + self.advance(); + if self.peek_char() == ' ' || self.peek_char() == '\n' { + while self.current < self.input.len() && self.current_char() != '\n' { + self.advance(); + } + self.next_token() + } + else { + return Some(self.build_token(start, TokenTypes::Error)); + } + } + else { Some(self.build_token(start, TokenTypes::Minus)) } } - '/' => Some(self.build_token(start, TokenTypes::Divide)), + '/' => { + if self.peek_char() == '*' { + self.read_block_comment(start) + } else { + Some(self.build_token(start, TokenTypes::Divide)) + } + } '%' => Some(self.build_token(start, TokenTypes::Modulo)), '=' => Some(self.build_token(start, TokenTypes::Equals)), '!' => { diff --git a/tests/crud_test.rs b/tests/crud_test.rs index 4dfc3e1..37fde89 100644 --- a/tests/crud_test.rs +++ b/tests/crud_test.rs @@ -132,6 +132,7 @@ fn test_drop_table() { fn test_alter_table() { let mut database = Database::new(); let sql = " + /* These should all succeed */ CREATE TABLE users ( id INTEGER, name TEXT @@ -142,7 +143,7 @@ fn test_alter_table() { ALTER TABLE new_users ADD COLUMN new_column INTEGER; ALTER TABLE new_users DROP COLUMN id; SELECT * FROM new_users; - + -- These should all fail ALTER TABLE users RENAME TO new_users; ALTER TABLE new_users DROP COLUMN id; ALTER TABLE new_users ADD COLUMN new_column INTEGER; @@ -159,10 +160,10 @@ fn test_alter_table() { assert_eq!(expected, *row); let expected_errors = vec![ - "Execution Error with statement starting on line 13 \n Error: Table `users` does not exist", - "Execution Error with statement starting on line 14 \n Error: Column `id` does not exist in table `new_users`", - "Execution Error with statement starting on line 15 \n Error: Column `new_column` already exists in table `new_users`", - "Execution Error with statement starting on line 16 \n Error: Column `id` does not exist in table `new_users`", + "Execution Error with statement starting on line 14 \n Error: Table `users` does not exist", + "Execution Error with statement starting on line 15 \n Error: Column `id` does not exist in table `new_users`", + "Execution Error with statement starting on line 16 \n Error: Column `new_column` already exists in table `new_users`", + "Execution Error with statement starting on line 17 \n Error: Column `id` does not exist in table `new_users`", ]; assert!(result[7..=10].iter().all(|result| result.is_err()));