Skip to content

Commit

Permalink
Refactor tokenizer to accomodate blocks
Browse files Browse the repository at this point in the history
  • Loading branch information
@ejpbruel2
ejpbruel2 committed Aug 31, 2021
1 parent 7451701 commit 2a05707
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 68 deletions.
12 changes: 10 additions & 2 deletions examples/cmdline_example/src/main.rs
View file
Expand Up @@ -58,8 +58,16 @@ fn main() {
lr.expand_all_documents(&mut errors);

println!("{}", lr.expanded[0]);

for msg in errors {

for msg in errors {
{
bla { bla di bla }

}
bla
bla

}
println!("{}\n", msg.to_live_file_error("", source));
}

Expand Down
33 changes: 14 additions & 19 deletions mace/src/code_editor.rs
View file
Expand Up @@ -8,8 +8,7 @@ use {
range_set::{RangeSet, Span},
size::Size,
text::Text,
token::{Keyword, Punctuator, TokenKind},
tokenizer::{Tokenizer, TokensByLine},
tokenizer::{Keyword, LineInfos, Punctuator, TokenKind, Tokenizer},
},
makepad_render::*,
makepad_widget::*,
Expand Down Expand Up @@ -111,11 +110,11 @@ impl CodeEditor {
self.apply_style(cx);
let visible_lines = self.visible_lines(cx, view_id, document.text.as_lines().len());
self.draw_selections(cx, &session.selections, &document.text, visible_lines);
self.draw_indent_guides(cx, document.tokenizer.tokens_by_line(), visible_lines);
self.draw_indent_guides(cx, document.tokenizer.line_infos(), visible_lines);
self.draw_text(
cx,
&document.text,
document.tokenizer.tokens_by_line(),
document.tokenizer.line_infos(),
visible_lines,
);
self.draw_carets(cx, &session.selections, &session.carets, visible_lines);
Expand Down Expand Up @@ -253,26 +252,22 @@ impl CodeEditor {
fn draw_indent_guides(
&mut self,
cx: &mut Cx,
tokens_by_line: TokensByLine<'_>,
line_infos: LineInfos<'_>,
visible_lines: VisibleLines,
) {
let origin = cx.get_turtle_pos();
let mut start_y = visible_lines.start_y;
for tokens in tokens_by_line {
let mut whitespace_len = 0;
for token in tokens {
if token.kind != TokenKind::Whitespace {
break;
}
whitespace_len += token.len;
}
for column in 0..whitespace_len {
for line_info in line_infos
.skip(visible_lines.start)
.take(visible_lines.end - visible_lines.start)
{
for block in line_info.blocks {
self.indent_guide.base.color = vec4(1.0, 1.0, 1.0, 1.0);
self.indent_guide.draw_quad_abs(
cx,
Rect {
pos: Vec2 {
x: origin.x + column as f32 * self.text_glyph_size.x,
x: origin.x + block.column as f32 * self.text_glyph_size.x,
y: start_y,
},
size: self.text_glyph_size,
Expand All @@ -287,22 +282,22 @@ impl CodeEditor {
&mut self,
cx: &mut Cx,
text: &Text,
tokens_by_line: TokensByLine<'_>,
line_infos: LineInfos<'_>,
visible_lines: VisibleLines,
) {
let origin = cx.get_turtle_pos();
let mut start_y = visible_lines.start_y;
for (line, tokens) in text
for (line, line_info) in text
.as_lines()
.iter()
.zip(tokens_by_line)
.zip(line_infos)
.skip(visible_lines.start)
.take(visible_lines.end - visible_lines.start)
{
let end_y = start_y + self.text_glyph_size.y;
let mut start_x = origin.x;
let mut start = 0;
let mut token_iter = tokens.iter();
let mut token_iter = line_info.tokens.iter();
let mut token_slot = token_iter.next();
while let Some(token) = token_slot {
let next_token = token_iter.next();
Expand Down
1 change: 0 additions & 1 deletion mace/src/lib.rs
View file
Expand Up @@ -17,6 +17,5 @@ pub mod size;
pub mod splitter;
pub mod tab_bar;
pub mod text;
pub mod token;
pub mod tokenizer;
pub mod tree_logic;
31 changes: 0 additions & 31 deletions mace/src/token.rs
View file

This file was deleted.

93 changes: 78 additions & 15 deletions mace/src/tokenizer.rs
View file
Expand Up @@ -2,7 +2,6 @@ use {
crate::{
delta::{Delta, OperationSpan},
text::Text,
token::{Keyword, Punctuator, Token, TokenKind},
},
std::slice::Iter,
};
Expand All @@ -20,8 +19,8 @@ impl Tokenizer {
tokenizer
}

pub fn tokens_by_line(&self) -> TokensByLine {
TokensByLine {
pub fn line_infos(&self) -> LineInfos<'_> {
LineInfos {
iter: self.lines.iter(),
}
}
Expand Down Expand Up @@ -55,8 +54,9 @@ impl Tokenizer {

pub fn refresh_cache(&mut self, text: &Text) {
let mut state = State::Initial(InitialState);
for (line, line_data) in self.lines.iter_mut().enumerate() {
match line_data {
let mut blocks = Vec::new();
for (line_index, line) in self.lines.iter_mut().enumerate() {
match line {
Some(Line {
start_state,
end_state,
Expand All @@ -67,7 +67,7 @@ impl Tokenizer {
_ => {
let start_state = state;
let mut tokens = Vec::new();
let mut cursor = Cursor::new(&text.as_lines()[line]);
let mut cursor = Cursor::new(&text.as_lines()[line_index]);
loop {
let (next_state, token) = state.next(&mut cursor);
state = next_state;
Expand All @@ -76,33 +76,84 @@ impl Tokenizer {
None => break,
}
}
*line_data = Some(Line {
*line = Some(Line {
start_state,
end_state: state,
tokens,
blocks: blocks.clone(),
});
}
}
}
}
}

pub struct TokensByLine<'a> {
pub struct LineInfos<'a> {
iter: Iter<'a, Option<Line>>,
}

impl<'a> Iterator for TokensByLine<'a> {
type Item = &'a Vec<Token>;
impl<'a> Iterator for LineInfos<'a> {
type Item = LineInfo<'a>;

fn next(&mut self) -> Option<&'a Vec<Token>> {
Some(&self.iter.next()?.as_ref().unwrap().tokens)
fn next(&mut self) -> Option<LineInfo<'a>> {
let line = self.iter.next()?.as_ref().unwrap();
Some(LineInfo {
tokens: &line.tokens,
blocks: &line.blocks,
})
}
}

pub struct LineInfo<'a> {
pub tokens: &'a [Token],
pub blocks: &'a [Block],
}

#[derive(Clone, Copy, Debug)]
pub struct Token {
pub len: usize,
pub kind: TokenKind,
}

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum TokenKind {
Comment,
Identifier,
Keyword(Keyword),
Number,
Punctuator(Punctuator),
String,
Whitespace,
Unknown,
}

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum Keyword {
Branch,
Loop,
Other,
}

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum Punctuator {
LeftParen,
RightParen,
LeftBrace,
RightBrace,
Other,
}

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Block {
pub column: usize,
pub keyword: Keyword,
}

struct Line {
start_state: State,
end_state: State,
tokens: Vec<Token>,
blocks: Vec<Block>,
}

#[derive(Clone, Copy, PartialEq)]
Expand Down Expand Up @@ -204,6 +255,20 @@ impl InitialState {
TokenKind::Punctuator(Punctuator::RightParen),
)
}
('{', _, _) => {
cursor.skip(1);
(
State::Initial(InitialState),
TokenKind::Punctuator(Punctuator::LeftBrace),
)
}
('}', _, _) => {
cursor.skip(1);
(
State::Initial(InitialState),
TokenKind::Punctuator(Punctuator::RightBrace),
)
}
('!', _, _)
| ('#', _, _)
| ('$', _, _)
Expand All @@ -226,9 +291,7 @@ impl InitialState {
| (']', _, _)
| ('^', _, _)
| ('_', _, _)
| ('{', _, _)
| ('|', _, _)
| ('}', _, _) => {
| ('|', _, _) => {
cursor.skip(1);
(
State::Initial(InitialState),
Expand Down

0 comments on commit 2a05707

Please sign in to comment.