Finish parser for ES2021

Cargo.toml

@@ -11,10 +11,3 @@ members = [
 [profile]
 release = { lto = true, codegen-units = 1 }
 bench = { lto = true, codegen-units = 1 }
-
-# Currently due to the amount of recursion and the size of vars, rslint-parse can stack overflow with larger files in debug mode
-# This may be adressed in the future, but for now we will either build with optimizations or use threads with a larger stack
-# FIXME: resolve stack overflows
-[profile.dev]
-debug = false
-opt-level = 3

rslint_lexer/Cargo.toml

@@ -9,7 +9,15 @@ license = "MIT"
 codespan-reporting = "0.9.5"
 rslint_syntax = { version = "0.1.0", path = "../rslint_syntax" }
 unicode-xid = "0.2.1"
+atty = { version = "0.2.14", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
 
 [dev-dependencies]
 quickcheck = "0.9"
 quickcheck_macros = "0.9"
+
+[features]
+highlight = ["atty", "ansi_term"]
+
+[package.metadata.docs.rs]
+features = ["highlight"]

rslint_lexer/src/highlight.rs

@@ -0,0 +1,121 @@
+use crate::*;
+
+#[cfg(feature = "highlight")]
+pub use ansi_term::{ANSIGenericString, Color};
+#[cfg(feature = "highlight")]
+use atty::is;
+
+/// A structure for syntax highlighting pieces of JavaScript source code
+/// using ANSI.
+///
+/// The highlighter will auto detect if stderr or stdout are terminals, if
+/// they are not then it will return the original uncolored source code.
+/// All errors encountered while lexing are ignored.
+///
+/// The highlighter is iterator based, which allows for coloring a part of code
+/// at a time.
+/// The highlighter's position can be controlled through various methods which allows
+/// for reuse of the highlighter without the need to rescan the source code
+#[cfg(feature = "highlight")]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Highlighter<'s> {
+    pub source: &'s str,
+    tokens: Vec<Token>,
+    /// Current token position
+    cur: usize,
+    /// Current byte index in source
+    cur_idx: usize,
+}
+
+#[cfg(feature = "highlight")]
+impl<'s> Highlighter<'s> {
+    /// Make a new highlighter, this will invoke the lexer to get tokens.
+    pub fn new(source: &'s str) -> Highlighter<'s> {
+        let tokens = Lexer::from_str(source, 0).map(|t| t.0).collect();
+
+        Self {
+            source,
+            tokens,
+            cur: 0,
+            cur_idx: 0,
+        }
+    }
+
+    fn check_terminal(&self) -> bool {
+        is(atty::Stream::Stderr) && is(atty::Stream::Stdout)
+    }
+
+    /// Reset the highlighter to the start of the source code
+    pub fn reset(&mut self) {}
+
+    /// Consume the rest of the highlighter's tokens and turn them into an ANSI colored string.
+    /// This returns an unaltered string if stdout and stderr are not terminals.
+    pub fn color(&mut self) -> String {
+        if !self.check_terminal() {
+            let ret = self.source[self.cur_idx..self.source.len()].to_string();
+            self.cur = self.tokens.len();
+            self.cur_idx = self.source.len();
+            return ret;
+        }
+
+        self.map(|x| x.to_string()).collect()
+    }
+
+    fn src(&self) -> &'s str {
+        &self.source[self.cur_idx..self.cur_idx + self.tokens.get(self.cur).unwrap().len]
+    }
+}
+
+macro_rules! rgb {
+    ($r:expr, $g:expr, $b:expr) => {
+        Color::RGB($r, $g, $b)
+    };
+}
+
+const PURPLE_IDENT: [&'static str; 4] = ["let", "class", "await", "yield"];
+
+#[cfg(feature = "highlight")]
+impl<'s> Iterator for Highlighter<'s> {
+    /// An individual colored token, you can see the color used by checking the string's style foreground
+    type Item = ANSIGenericString<'s, str>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.tokens.get(self.cur) == None {
+            return None;
+        }
+
+        let color = match self.tokens.get(self.cur)?.kind {
+            T!['{'] | T!['}'] | T!['('] | T![')'] => rgb![255, 215, 0],
+            T![import] => rgb![97, 175, 239],
+            T![ident] if PURPLE_IDENT.contains(&self.src()) => rgb![198, 120, 221],
+            T![ident] if self.src() == "from" => rgb![97, 175, 239],
+            T![ident] if self.src() == "Math" => rgb![229, 192, 123],
+            T![ident] => rgb![224, 108, 117],
+            T![instanceof] | T![new] | T![?] | T![delete] | T![:] | T![const] => {
+                rgb![198, 120, 221]
+            }
+            t if t.is_punct() => rgb![86, 182, 194],
+            t if t.is_keyword() => rgb![198, 120, 221],
+            SyntaxKind::STRING | SyntaxKind::BACKTICK | SyntaxKind::TEMPLATE_CHUNK => {
+                rgb![152, 195, 121]
+            }
+            SyntaxKind::NUMBER => rgb![209, 154, 102],
+            SyntaxKind::DOLLARCURLY => rgb![198, 120, 221],
+            SyntaxKind::ERROR_TOKEN => rgb![244, 71, 71],
+            SyntaxKind::COMMENT => rgb![127, 132, 142],
+            _ => Color::White,
+        };
+
+        let string = self.src();
+        self.cur_idx += self.tokens.get(self.cur).unwrap().len;
+        self.cur += 1;
+        Some(color.paint(string))
+    }
+}
+
+/// Colors a piece of source code using ANSI.  
+/// The string returned will be unaltered if stdout and stderr are not terminals.
+#[cfg(feature = "highlight")]
+pub fn color(source: &str) -> String {
+    Highlighter::new(source).color()
+}

rslint_lexer/src/labels.rs

@@ -1,6 +1,16 @@
 use crate::{Lexer, SyntaxKind, T};
 
 impl Lexer<'_> {
+    #[inline]
+    pub(crate) fn resolve_label_a(&mut self) -> Option<SyntaxKind> {
+        if let Some(b"wait") = self.bytes.get(self.cur + 1..self.cur + 5) {
+            self.advance(4);
+            Some(T![await])
+        } else {
+            None
+        }
+    }
+
     #[inline]
     pub(crate) fn resolve_label_b(&mut self) -> Option<SyntaxKind> {
         match self.bytes.get(self.cur..(self.cur + 5)) {
@@ -226,15 +236,15 @@ impl Lexer<'_> {
                 } else {
                     None
                 }
-            },
+            }
             Some(b'e') => {
                 if let Some(b'w') = self.bytes.get(self.cur + 2) {
                     self.advance(2);
                     Some(T![new])
                 } else {
                     None
                 }
-            },
+            }
             _ => None,
         }
     }
@@ -276,23 +286,21 @@ impl Lexer<'_> {
     #[inline]
     pub(crate) fn resolve_label_t(&mut self) -> Option<SyntaxKind> {
         match self.bytes.get(self.cur + 1) {
-            Some(b'r') => {
-                match self.bytes.get(self.cur + 2) {
-                    Some(b'y') => {
-                        self.advance(2);
-                        Some(T![try])
-                    }
-                    Some(b'u') => {
-                        if let Some(b'e') = self.bytes.get(self.cur + 3) {
-                            self.advance(3);
-                            Some(T![true])
-                        } else {
-                            None
-                        }
+            Some(b'r') => match self.bytes.get(self.cur + 2) {
+                Some(b'y') => {
+                    self.advance(2);
+                    Some(T![try])
+                }
+                Some(b'u') => {
+                    if let Some(b'e') = self.bytes.get(self.cur + 3) {
+                        self.advance(3);
+                        Some(T![true])
+                    } else {
+                        None
                     }
-                    _ => None,
                 }
-            }
+                _ => None,
+            },
             Some(b'h') => match self.bytes.get(self.cur + 2) {
                 Some(b'i') => {
                     if let Some(b's') = self.bytes.get(self.cur + 3) {