add tests to PR #529

sqlparser-rs · Jul 7, 2022 · 6d32cca · 6d32cca
1 parent f668e25
commit 6d32cca
Showing 1 changed file with 16 additions and 0 deletions.
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -1212,6 +1212,22 @@ mod tests {
         );
     }
 
+    #[test]
+    fn tokenize_unterminated_string_literal_utf8() {
+        let sql = String::from("SELECT \"なにか\" FROM Y WHERE \"なにか\" = 'test;");
+
+        let dialect = GenericDialect {};
+        let mut tokenizer = Tokenizer::new(&dialect, &sql);
+        assert_eq!(
+            tokenizer.tokenize(),
+            Err(TokenizerError {
+                message: "Unterminated string literal".to_string(),
+                line: 1,
+                col: 35
+            })
+        );
+    }
+
     #[test]
     fn tokenize_invalid_string_cols() {
         let sql = String::from("\n\nSELECT * FROM table\tمصطفىh");