diff --git a/examples/cmdline_example/src/main.rs b/examples/cmdline_example/src/main.rs index c9dfe87eb..f4caf5f0f 100644 --- a/examples/cmdline_example/src/main.rs +++ b/examples/cmdline_example/src/main.rs @@ -58,8 +58,16 @@ fn main() { lr.expand_all_documents(&mut errors); println!("{}", lr.expanded[0]); - - for msg in errors { + + for msg in errors { + { + bla { bla di bla } + + } + bla + bla + + } println!("{}\n", msg.to_live_file_error("", source)); } diff --git a/mace/src/code_editor.rs b/mace/src/code_editor.rs index e231b0b54..17a049c3f 100644 --- a/mace/src/code_editor.rs +++ b/mace/src/code_editor.rs @@ -8,8 +8,7 @@ use { range_set::{RangeSet, Span}, size::Size, text::Text, - token::{Keyword, Punctuator, TokenKind}, - tokenizer::{Tokenizer, TokensByLine}, + tokenizer::{Keyword, LineInfos, Punctuator, TokenKind, Tokenizer}, }, makepad_render::*, makepad_widget::*, @@ -111,11 +110,11 @@ impl CodeEditor { self.apply_style(cx); let visible_lines = self.visible_lines(cx, view_id, document.text.as_lines().len()); self.draw_selections(cx, &session.selections, &document.text, visible_lines); - self.draw_indent_guides(cx, document.tokenizer.tokens_by_line(), visible_lines); + self.draw_indent_guides(cx, document.tokenizer.line_infos(), visible_lines); self.draw_text( cx, &document.text, - document.tokenizer.tokens_by_line(), + document.tokenizer.line_infos(), visible_lines, ); self.draw_carets(cx, &session.selections, &session.carets, visible_lines); @@ -253,26 +252,22 @@ impl CodeEditor { fn draw_indent_guides( &mut self, cx: &mut Cx, - tokens_by_line: TokensByLine<'_>, + line_infos: LineInfos<'_>, visible_lines: VisibleLines, ) { let origin = cx.get_turtle_pos(); let mut start_y = visible_lines.start_y; - for tokens in tokens_by_line { - let mut whitespace_len = 0; - for token in tokens { - if token.kind != TokenKind::Whitespace { - break; - } - whitespace_len += token.len; - } - for column in 0..whitespace_len { + for line_info in line_infos + .skip(visible_lines.start) + .take(visible_lines.end - visible_lines.start) + { + for block in line_info.blocks { self.indent_guide.base.color = vec4(1.0, 1.0, 1.0, 1.0); self.indent_guide.draw_quad_abs( cx, Rect { pos: Vec2 { - x: origin.x + column as f32 * self.text_glyph_size.x, + x: origin.x + block.column as f32 * self.text_glyph_size.x, y: start_y, }, size: self.text_glyph_size, @@ -287,22 +282,22 @@ impl CodeEditor { &mut self, cx: &mut Cx, text: &Text, - tokens_by_line: TokensByLine<'_>, + line_infos: LineInfos<'_>, visible_lines: VisibleLines, ) { let origin = cx.get_turtle_pos(); let mut start_y = visible_lines.start_y; - for (line, tokens) in text + for (line, line_info) in text .as_lines() .iter() - .zip(tokens_by_line) + .zip(line_infos) .skip(visible_lines.start) .take(visible_lines.end - visible_lines.start) { let end_y = start_y + self.text_glyph_size.y; let mut start_x = origin.x; let mut start = 0; - let mut token_iter = tokens.iter(); + let mut token_iter = line_info.tokens.iter(); let mut token_slot = token_iter.next(); while let Some(token) = token_slot { let next_token = token_iter.next(); diff --git a/mace/src/lib.rs b/mace/src/lib.rs index f2acb29c5..f2a2f8632 100644 --- a/mace/src/lib.rs +++ b/mace/src/lib.rs @@ -17,6 +17,5 @@ pub mod size; pub mod splitter; pub mod tab_bar; pub mod text; -pub mod token; pub mod tokenizer; pub mod tree_logic; diff --git a/mace/src/token.rs b/mace/src/token.rs deleted file mode 100644 index 7f137162d..000000000 --- a/mace/src/token.rs +++ /dev/null @@ -1,31 +0,0 @@ -#[derive(Clone, Copy, Debug)] -pub struct Token { - pub len: usize, - pub kind: TokenKind, -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub enum TokenKind { - Comment, - Identifier, - Keyword(Keyword), - Number, - Punctuator(Punctuator), - String, - Whitespace, - Unknown, -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub enum Keyword { - Branch, - Loop, - Other, -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub enum Punctuator { - LeftParen, - RightParen, - Other, -} diff --git a/mace/src/tokenizer.rs b/mace/src/tokenizer.rs index b4c216dee..2b772233b 100644 --- a/mace/src/tokenizer.rs +++ b/mace/src/tokenizer.rs @@ -2,7 +2,6 @@ use { crate::{ delta::{Delta, OperationSpan}, text::Text, - token::{Keyword, Punctuator, Token, TokenKind}, }, std::slice::Iter, }; @@ -20,8 +19,8 @@ impl Tokenizer { tokenizer } - pub fn tokens_by_line(&self) -> TokensByLine { - TokensByLine { + pub fn line_infos(&self) -> LineInfos<'_> { + LineInfos { iter: self.lines.iter(), } } @@ -55,8 +54,9 @@ impl Tokenizer { pub fn refresh_cache(&mut self, text: &Text) { let mut state = State::Initial(InitialState); - for (line, line_data) in self.lines.iter_mut().enumerate() { - match line_data { + let mut blocks = Vec::new(); + for (line_index, line) in self.lines.iter_mut().enumerate() { + match line { Some(Line { start_state, end_state, @@ -67,7 +67,7 @@ impl Tokenizer { _ => { let start_state = state; let mut tokens = Vec::new(); - let mut cursor = Cursor::new(&text.as_lines()[line]); + let mut cursor = Cursor::new(&text.as_lines()[line_index]); loop { let (next_state, token) = state.next(&mut cursor); state = next_state; @@ -76,10 +76,11 @@ impl Tokenizer { None => break, } } - *line_data = Some(Line { + *line = Some(Line { start_state, end_state: state, tokens, + blocks: blocks.clone(), }); } } @@ -87,22 +88,72 @@ impl Tokenizer { } } -pub struct TokensByLine<'a> { +pub struct LineInfos<'a> { iter: Iter<'a, Option>, } -impl<'a> Iterator for TokensByLine<'a> { - type Item = &'a Vec; +impl<'a> Iterator for LineInfos<'a> { + type Item = LineInfo<'a>; - fn next(&mut self) -> Option<&'a Vec> { - Some(&self.iter.next()?.as_ref().unwrap().tokens) + fn next(&mut self) -> Option> { + let line = self.iter.next()?.as_ref().unwrap(); + Some(LineInfo { + tokens: &line.tokens, + blocks: &line.blocks, + }) } } +pub struct LineInfo<'a> { + pub tokens: &'a [Token], + pub blocks: &'a [Block], +} + +#[derive(Clone, Copy, Debug)] +pub struct Token { + pub len: usize, + pub kind: TokenKind, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum TokenKind { + Comment, + Identifier, + Keyword(Keyword), + Number, + Punctuator(Punctuator), + String, + Whitespace, + Unknown, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum Keyword { + Branch, + Loop, + Other, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum Punctuator { + LeftParen, + RightParen, + LeftBrace, + RightBrace, + Other, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct Block { + pub column: usize, + pub keyword: Keyword, +} + struct Line { start_state: State, end_state: State, tokens: Vec, + blocks: Vec, } #[derive(Clone, Copy, PartialEq)] @@ -204,6 +255,20 @@ impl InitialState { TokenKind::Punctuator(Punctuator::RightParen), ) } + ('{', _, _) => { + cursor.skip(1); + ( + State::Initial(InitialState), + TokenKind::Punctuator(Punctuator::LeftBrace), + ) + } + ('}', _, _) => { + cursor.skip(1); + ( + State::Initial(InitialState), + TokenKind::Punctuator(Punctuator::RightBrace), + ) + } ('!', _, _) | ('#', _, _) | ('$', _, _) @@ -226,9 +291,7 @@ impl InitialState { | (']', _, _) | ('^', _, _) | ('_', _, _) - | ('{', _, _) - | ('|', _, _) - | ('}', _, _) => { + | ('|', _, _) => { cursor.skip(1); ( State::Initial(InitialState),