+1 −1

noop-tokenize.rs

examples/noop-tokenize.rs

+1 −1

tokenize.rs

examples/tokenize.rs

+1 −1

interface.rs

src/tokenizer/interface.rs

+129 −2

mod.rs

src/tokenizer/mod.rs

+1 −1

mod.rs

src/tree_builder/mod.rs

+3 −3

tokenizer.rs

tests/tokenizer.rs