Merge pull request #292 from epage/unicode

perf(parser): Auto-detect unicode
crate-ci · Jun 29, 2021 · a46cc76 · a46cc76
2 parents 21231bf + ded90f2
commit a46cc76
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/typos/Cargo.toml b/crates/typos/Cargo.toml
@@ -25,3 +25,4 @@ simdutf8 = "0.1.1"
 itertools = "0.10"
 log = "0.4"
 unicode-segmentation = "1.7.1"
+bstr = "0.2"
diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs
@@ -1,3 +1,5 @@
+use bstr::ByteSlice;
+
 /// Define rules for tokenizaing a buffer.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct TokenizerBuilder {
@@ -67,7 +69,7 @@ impl Tokenizer {
     }
 
     pub fn parse_str<'c>(&'c self, content: &'c str) -> impl Iterator<Item = Identifier<'c>> {
-        let iter = if self.unicode {
+        let iter = if self.unicode && !ByteSlice::is_ascii(content.as_bytes()) {
             itertools::Either::Left(unicode_parser::iter_literals(content))
         } else {
             itertools::Either::Right(ascii_parser::iter_literals(content.as_bytes()))
@@ -79,7 +81,7 @@ impl Tokenizer {
     }
 
     pub fn parse_bytes<'c>(&'c self, content: &'c [u8]) -> impl Iterator<Item = Identifier<'c>> {
-        let iter = if self.unicode {
+        let iter = if self.unicode && !ByteSlice::is_ascii(content) {
             let iter = Utf8Chunks::new(content).flat_map(move |c| unicode_parser::iter_literals(c));
             itertools::Either::Left(iter)
         } else {