feat(parse): Make identifier symbols configurable

Cargo.toml

@@ -36,6 +36,7 @@ unicase = "1.1"
 bstr = "0.2"
 log = "0.4"
 env_logger = "0.6"
+unicode-segmentation = "1.3.0"
 
 [dev-dependencies]
 assert_fs = "0.10"

src/tokens.rs

@@ -10,6 +10,7 @@ pub enum Case {
 pub struct ParserBuilder {
     ignore_hex: bool,
     include_digits: bool,
+    include_chars: String,
 }
 
 impl ParserBuilder {
@@ -27,11 +28,22 @@ impl ParserBuilder {
         self
     }
 
+    pub fn include_chars(&mut self, chars: String) -> &mut Self {
+        self.include_chars = chars;
+        self
+    }
+
     pub fn build(&self) -> Parser {
-        let mut pattern = r#"\b(\p{Alphabetic}|_|'"#.to_owned();
+        let mut pattern = r#"\b(\p{Alphabetic}"#.to_owned();
         if self.include_digits {
             pattern.push_str(r#"|\d"#);
         }
+        for grapheme in
+            unicode_segmentation::UnicodeSegmentation::graphemes(self.include_chars.as_str(), true)
+        {
+            let escaped = regex::escape(&grapheme);
+            pattern.push_str(&format!("|{}", escaped));
+        }
         pattern.push_str(r#")+\b"#);
         let words_str = regex::Regex::new(&pattern).unwrap();
         let words_bytes = regex::bytes::Regex::new(&pattern).unwrap();
@@ -48,6 +60,7 @@ impl Default for ParserBuilder {
         Self {
             ignore_hex: true,
             include_digits: true,
+            include_chars: "_'".to_owned(),
         }
     }
 }