+7 −4

noop-tokenize.rs

examples/noop-tokenize.rs

+7 −4

tokenize.rs

examples/tokenize.rs

+7 −3

driver.rs

src/driver.rs

+1 −1

lib.rs

src/lib.rs

+6 −1

buffer_queue.rs

src/tokenizer/buffer_queue.rs

+76 −38

mod.rs

src/tokenizer/char_ref/mod.rs

+129 −118

mod.rs

src/tokenizer/mod.rs

+5 −1

tokenizer.rs

tests/tokenizer.rs