Support dollar mark in identifer

kawasin73 · Aug 9, 2023 · 058035c · 058035c
1 parent 31416e4
commit 058035c
Showing 1 changed file with 11 additions and 6 deletions.
diff --git a/src/token.rs b/src/token.rs
@@ -20,24 +20,25 @@ const CHAR_X: u8 = 0x00;
 const CHAR_ALPHABET: u8 = 0x01;
 const CHAR_UNDERSCORE: u8 = 0x02;
 const CHAR_DIGIT: u8 = 0x03;
-const CHAR_QUOTE: u8 = 0x04; // ", ', `
-const CHAR_QUOTE2: u8 = 0x05; // [
+const CHAR_DOLLAR: u8 = 0x04;
+const CHAR_QUOTE: u8 = 0x05; // ", ', `
+const CHAR_QUOTE2: u8 = 0x06; // [
 const CHAR_INVALID: u8 = 0xFF;
 
 static CHAR_LOOKUP_TABLE: [u8; 256] = [
     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00 - 0x07
     b' ', b' ', 0xFF, b' ', b' ', 0xFF, 0xFF, 0xFF, // 0x08 - 0x0F
     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10 - 0x17
     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x18 - 0x1F
-    b' ', b'!', 0x04, 0xFF, 0xFF, 0xFF, 0xFF, 0x04, // 0x20 - 0x27
+    b' ', b'!', 0x05, 0xFF, 0x04, 0xFF, 0xFF, 0x05, // 0x20 - 0x27
     b'(', b')', b'*', b'+', b',', b'-', b'.', 0xFF, // 0x28 - 0x2F
     0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0x30 - 0x37
     0x03, 0x03, 0xFF, b';', b'<', b'=', b'>', 0xFF, // 0x38 - 0x3F
     0xFF, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0x40 - 0x47
     0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0x48 - 0x4F
     0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0x50 - 0x57
-    0x00, 0x01, 0x01, 0x05, 0xFF, 0xFF, 0xFF, 0x02, // 0x58 - 0x5F
-    0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0x60 - 0x67
+    0x00, 0x01, 0x01, 0x06, 0xFF, 0xFF, 0xFF, 0x02, // 0x58 - 0x5F
+    0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0x60 - 0x67
     0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0x68 - 0x6F
     0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0x70 - 0x77
     0x00, 0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x78 - 0x7F
@@ -291,7 +292,7 @@ pub fn get_token(input: &[u8]) -> Option<(usize, Token)> {
 fn len_identifier(input_bytes: &[u8]) -> usize {
     // Skip the first byte since it is already checked.
     for (i, &byte) in input_bytes.iter().skip(1).enumerate() {
-        if CHAR_LOOKUP_TABLE[byte as usize] > CHAR_DIGIT {
+        if CHAR_LOOKUP_TABLE[byte as usize] > CHAR_DOLLAR {
             return i + 1;
         }
     }
@@ -509,6 +510,10 @@ mod tests {
             get_token(b"_ "),
             Some((1, Token::Identifier(b"_".as_slice().into())))
         );
+        assert_eq!(
+            get_token(b"hello$ "),
+            Some((6, Token::Identifier(b"hello$".as_slice().into())))
+        );
     }
 
     #[test]