Skip to content

Commit

Permalink
add codegen tests for LUT
Browse files Browse the repository at this point in the history
  • Loading branch information
@RustyYato
RustyYato committed May 3, 2024
1 parent 9149056 commit 0dfcf00
Show file tree
Hide file tree
Showing 7 changed files with 279 additions and 38 deletions.
195 changes: 195 additions & 0 deletions logos-cli/tests/data/no_error_lut/fmt_output.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
#[derive()]
enum Token {
Newline,
AnyUnicode,
Any,
}
impl<'s> ::logos::Logos<'s> for Token {
type Error = ();
type Extras = ();
type Source = [u8];
fn lex(lex: &mut ::logos::Lexer<'s, Self>) {
use logos::internal::{CallbackResult, LexerInternal};
type Lexer<'s> = ::logos::Lexer<'s, Token>;
fn _end<'s>(lex: &mut Lexer<'s>) {
lex.end()
}
fn _error<'s>(lex: &mut Lexer<'s>) {
lex.bump_unchecked(1);
lex.error();
}
macro_rules ! _fast_loop { ($ lex : ident , $ test : ident , $ miss : expr) => { while let Some (arr) = $ lex . read :: < & [u8 ; 16] > () { if $ test (arr [0]) { if $ test (arr [1]) { if $ test (arr [2]) { if $ test (arr [3]) { if $ test (arr [4]) { if $ test (arr [5]) { if $ test (arr [6]) { if $ test (arr [7]) { if $ test (arr [8]) { if $ test (arr [9]) { if $ test (arr [10]) { if $ test (arr [11]) { if $ test (arr [12]) { if $ test (arr [13]) { if $ test (arr [14]) { if $ test (arr [15]) { $ lex . bump_unchecked (16) ; continue ; } $ lex . bump_unchecked (15) ; return $ miss ; } $ lex . bump_unchecked (14) ; return $ miss ; } $ lex . bump_unchecked (13) ; return $ miss ; } $ lex . bump_unchecked (12) ; return $ miss ; } $ lex . bump_unchecked (11) ; return $ miss ; } $ lex . bump_unchecked (10) ; return $ miss ; } $ lex . bump_unchecked (9) ; return $ miss ; } $ lex . bump_unchecked (8) ; return $ miss ; } $ lex . bump_unchecked (7) ; return $ miss ; } $ lex . bump_unchecked (6) ; return $ miss ; } $ lex . bump_unchecked (5) ; return $ miss ; } $ lex . bump_unchecked (4) ; return $ miss ; } $ lex . bump_unchecked (3) ; return $ miss ; } $ lex . bump_unchecked (2) ; return $ miss ; } $ lex . bump_unchecked (1) ; return $ miss ; } return $ miss ; } while $ lex . test ($ test) { $ lex . bump_unchecked (1) ; } $ miss } ; }
#[inline]
fn goto1_x<'s>(lex: &mut Lexer<'s>) {
lex.set(Ok(Token::Newline));
}
#[inline]
fn goto11_ctx11_x<'s>(lex: &mut Lexer<'s>) {
lex.set(Ok(Token::Any));
}
#[inline]
fn goto2_ctx11_x<'s>(lex: &mut Lexer<'s>) {
lex.set(Ok(Token::AnyUnicode));
}
#[inline]
fn goto16_ctx11_x<'s>(lex: &mut Lexer<'s>) {
match lex.read::<&[u8; 2usize]>() {
Some([128u8..=159u8, 128u8..=191u8]) => {
lex.bump_unchecked(2usize);
goto2_ctx11_x(lex)
}
_ => goto11_ctx11_x(lex),
}
}
#[inline]
fn goto17_ctx11_x<'s>(lex: &mut Lexer<'s>) {
match lex.read::<&[u8; 3usize]>() {
Some([144u8..=191u8, 128u8..=191u8, 128u8..=191u8]) => {
lex.bump_unchecked(3usize);
goto2_ctx11_x(lex)
}
_ => goto11_ctx11_x(lex),
}
}
#[inline]
fn goto2_x<'s>(lex: &mut Lexer<'s>) {
lex.set(Ok(Token::AnyUnicode));
}
#[inline]
fn goto13_ctx11_x<'s>(lex: &mut Lexer<'s>) {
match lex.read::<&[u8; 1usize]>() {
Some([128u8..=191u8]) => {
lex.bump_unchecked(1usize);
goto2_ctx11_x(lex)
}
_ => goto11_ctx11_x(lex),
}
}
#[inline]
fn goto18_ctx11_x<'s>(lex: &mut Lexer<'s>) {
match lex.read::<&[u8; 3usize]>() {
Some([128u8..=191u8, 128u8..=191u8, 128u8..=191u8]) => {
lex.bump_unchecked(3usize);
goto2_ctx11_x(lex)
}
_ => goto11_ctx11_x(lex),
}
}
#[inline]
fn goto15_ctx11_x<'s>(lex: &mut Lexer<'s>) {
match lex.read::<&[u8; 2usize]>() {
Some([128u8..=191u8, 128u8..=191u8]) => {
lex.bump_unchecked(2usize);
goto2_ctx11_x(lex)
}
_ => goto11_ctx11_x(lex),
}
}
#[inline]
fn goto14_ctx11_x<'s>(lex: &mut Lexer<'s>) {
match lex.read::<&[u8; 2usize]>() {
Some([160u8..=191u8, 128u8..=191u8]) => {
lex.bump_unchecked(2usize);
goto2_ctx11_x(lex)
}
_ => goto11_ctx11_x(lex),
}
}
#[inline]
fn goto19_ctx11_x<'s>(lex: &mut Lexer<'s>) {
match lex.read::<&[u8; 3usize]>() {
Some([128u8..=143u8, 128u8..=191u8, 128u8..=191u8]) => {
lex.bump_unchecked(3usize);
goto2_ctx11_x(lex)
}
_ => goto11_ctx11_x(lex),
}
}
#[inline]
fn goto11_x<'s>(lex: &mut Lexer<'s>) {
lex.set(Ok(Token::Any));
}
#[inline]
fn goto20<'s>(lex: &mut Lexer<'s>) {
enum Jump {
J1,
J16,
J17,
J2,
J13,
J18,
J15,
J14,
J19,
J11,
}
const LUT: [Jump; 256] = {
use Jump::*;
[
J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J1, J2, J2, J2, J2, J2, J2, J2, J2, J2,
J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2,
J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2,
J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2,
J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2,
J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2, J2,
J2, J2, J2, J2, J2, J2, J2, J2, J11, J11, J11, J11, J11, J11, J11, J11, J11,
J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11,
J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11,
J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11, J11,
J11, J11, J11, J11, J11, J11, J11, J11, J11, J13, J13, J13, J13, J13, J13, J13,
J13, J13, J13, J13, J13, J13, J13, J13, J13, J13, J13, J13, J13, J13, J13, J13,
J13, J13, J13, J13, J13, J13, J13, J14, J15, J15, J15, J15, J15, J15, J15, J15,
J15, J15, J15, J15, J16, J15, J15, J17, J18, J18, J18, J19, J11, J11, J11, J11,
J11, J11, J11, J11, J11, J11, J11,
]
};
let byte = match lex.read::<u8>() {
Some(byte) => byte,
None => return _end(lex),
};
match LUT[byte as usize] {
Jump::J1 => {
lex.bump_unchecked(1usize);
goto1_x(lex)
}
Jump::J16 => {
lex.bump_unchecked(1usize);
goto16_ctx11_x(lex)
}
Jump::J17 => {
lex.bump_unchecked(1usize);
goto17_ctx11_x(lex)
}
Jump::J2 => {
lex.bump_unchecked(1usize);
goto2_x(lex)
}
Jump::J13 => {
lex.bump_unchecked(1usize);
goto13_ctx11_x(lex)
}
Jump::J18 => {
lex.bump_unchecked(1usize);
goto18_ctx11_x(lex)
}
Jump::J15 => {
lex.bump_unchecked(1usize);
goto15_ctx11_x(lex)
}
Jump::J14 => {
lex.bump_unchecked(1usize);
goto14_ctx11_x(lex)
}
Jump::J19 => {
lex.bump_unchecked(1usize);
goto19_ctx11_x(lex)
}
Jump::J11 => {
lex.bump_unchecked(1usize);
goto11_x(lex)
}
}
}
goto20(lex)
}
}
10 changes: 10 additions & 0 deletions logos-cli/tests/data/no_error_lut/input.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#[derive(Logos)]
#[logos(source = [u8])]
enum Token {
#[token("\n")]
Newline,
#[regex(".")]
AnyUnicode,
#[regex(b".", priority = 0)]
Any,
}
1 change: 1 addition & 0 deletions logos-cli/tests/data/no_error_lut/output.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# [derive ()] enum Token { Newline , AnyUnicode , Any , }impl < 's > :: logos :: Logos < 's > for Token { type Error = () ; type Extras = () ; type Source = [u8] ; fn lex (lex : & mut :: logos :: Lexer < 's , Self >) { use :: logos :: internal :: { LexerInternal , CallbackResult } ; type Lexer < 's > = :: logos :: Lexer < 's , Token > ; fn _end < 's > (lex : & mut Lexer < 's >) { lex . end () } fn _error < 's > (lex : & mut Lexer < 's >) { lex . bump_unchecked (1) ; lex . error () ; } macro_rules ! _fast_loop { ($ lex : ident , $ test : ident , $ miss : expr) => { while let Some (arr) = $ lex . read :: < & [u8 ; 16] > () { if $ test (arr [0]) { if $ test (arr [1]) { if $ test (arr [2]) { if $ test (arr [3]) { if $ test (arr [4]) { if $ test (arr [5]) { if $ test (arr [6]) { if $ test (arr [7]) { if $ test (arr [8]) { if $ test (arr [9]) { if $ test (arr [10]) { if $ test (arr [11]) { if $ test (arr [12]) { if $ test (arr [13]) { if $ test (arr [14]) { if $ test (arr [15]) { $ lex . bump_unchecked (16) ; continue ; } $ lex . bump_unchecked (15) ; return $ miss ; } $ lex . bump_unchecked (14) ; return $ miss ; } $ lex . bump_unchecked (13) ; return $ miss ; } $ lex . bump_unchecked (12) ; return $ miss ; } $ lex . bump_unchecked (11) ; return $ miss ; } $ lex . bump_unchecked (10) ; return $ miss ; } $ lex . bump_unchecked (9) ; return $ miss ; } $ lex . bump_unchecked (8) ; return $ miss ; } $ lex . bump_unchecked (7) ; return $ miss ; } $ lex . bump_unchecked (6) ; return $ miss ; } $ lex . bump_unchecked (5) ; return $ miss ; } $ lex . bump_unchecked (4) ; return $ miss ; } $ lex . bump_unchecked (3) ; return $ miss ; } $ lex . bump_unchecked (2) ; return $ miss ; } $ lex . bump_unchecked (1) ; return $ miss ; } return $ miss ; } while $ lex . test ($ test) { $ lex . bump_unchecked (1) ; } $ miss } ; } # [inline] fn goto1_x < 's > (lex : & mut Lexer < 's >) { lex . set (Ok (Token :: Newline)) ; } # [inline] fn goto11_ctx11_x < 's > (lex : & mut Lexer < 's >) { lex . set (Ok (Token :: Any)) ; } # [inline] fn goto2_ctx11_x < 's > (lex : & mut Lexer < 's >) { lex . set (Ok (Token :: AnyUnicode)) ; } # [inline] fn goto16_ctx11_x < 's > (lex : & mut Lexer < 's >) { match lex . read :: < & [u8 ; 2usize] > () { Some ([128u8 ..= 159u8 , 128u8 ..= 191u8]) => { lex . bump_unchecked (2usize) ; goto2_ctx11_x (lex) } , _ => goto11_ctx11_x (lex) , } } # [inline] fn goto17_ctx11_x < 's > (lex : & mut Lexer < 's >) { match lex . read :: < & [u8 ; 3usize] > () { Some ([144u8 ..= 191u8 , 128u8 ..= 191u8 , 128u8 ..= 191u8]) => { lex . bump_unchecked (3usize) ; goto2_ctx11_x (lex) } , _ => goto11_ctx11_x (lex) , } } # [inline] fn goto2_x < 's > (lex : & mut Lexer < 's >) { lex . set (Ok (Token :: AnyUnicode)) ; } # [inline] fn goto13_ctx11_x < 's > (lex : & mut Lexer < 's >) { match lex . read :: < & [u8 ; 1usize] > () { Some ([128u8 ..= 191u8]) => { lex . bump_unchecked (1usize) ; goto2_ctx11_x (lex) } , _ => goto11_ctx11_x (lex) , } } # [inline] fn goto18_ctx11_x < 's > (lex : & mut Lexer < 's >) { match lex . read :: < & [u8 ; 3usize] > () { Some ([128u8 ..= 191u8 , 128u8 ..= 191u8 , 128u8 ..= 191u8]) => { lex . bump_unchecked (3usize) ; goto2_ctx11_x (lex) } , _ => goto11_ctx11_x (lex) , } } # [inline] fn goto15_ctx11_x < 's > (lex : & mut Lexer < 's >) { match lex . read :: < & [u8 ; 2usize] > () { Some ([128u8 ..= 191u8 , 128u8 ..= 191u8]) => { lex . bump_unchecked (2usize) ; goto2_ctx11_x (lex) } , _ => goto11_ctx11_x (lex) , } } # [inline] fn goto14_ctx11_x < 's > (lex : & mut Lexer < 's >) { match lex . read :: < & [u8 ; 2usize] > () { Some ([160u8 ..= 191u8 , 128u8 ..= 191u8]) => { lex . bump_unchecked (2usize) ; goto2_ctx11_x (lex) } , _ => goto11_ctx11_x (lex) , } } # [inline] fn goto19_ctx11_x < 's > (lex : & mut Lexer < 's >) { match lex . read :: < & [u8 ; 3usize] > () { Some ([128u8 ..= 143u8 , 128u8 ..= 191u8 , 128u8 ..= 191u8]) => { lex . bump_unchecked (3usize) ; goto2_ctx11_x (lex) } , _ => goto11_ctx11_x (lex) , } } # [inline] fn goto11_x < 's > (lex : & mut Lexer < 's >) { lex . set (Ok (Token :: Any)) ; } # [inline] fn goto20 < 's > (lex : & mut Lexer < 's >) { enum Jump { J1 , J16 , J17 , J2 , J13 , J18 , J15 , J14 , J19 , J11 , } const LUT : [Jump ; 256] = { use Jump :: * ; [J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J1 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J2 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J13 , J14 , J15 , J15 , J15 , J15 , J15 , J15 , J15 , J15 , J15 , J15 , J15 , J15 , J16 , J15 , J15 , J17 , J18 , J18 , J18 , J19 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11 , J11] } ; let byte = match lex . read :: < u8 > () { Some (byte) => byte , None => return _end (lex) , } ; match LUT [byte as usize] { Jump :: J1 => { lex . bump_unchecked (1usize) ; goto1_x (lex) } , Jump :: J16 => { lex . bump_unchecked (1usize) ; goto16_ctx11_x (lex) } , Jump :: J17 => { lex . bump_unchecked (1usize) ; goto17_ctx11_x (lex) } , Jump :: J2 => { lex . bump_unchecked (1usize) ; goto2_x (lex) } , Jump :: J13 => { lex . bump_unchecked (1usize) ; goto13_ctx11_x (lex) } , Jump :: J18 => { lex . bump_unchecked (1usize) ; goto18_ctx11_x (lex) } , Jump :: J15 => { lex . bump_unchecked (1usize) ; goto15_ctx11_x (lex) } , Jump :: J14 => { lex . bump_unchecked (1usize) ; goto14_ctx11_x (lex) } , Jump :: J19 => { lex . bump_unchecked (1usize) ; goto19_ctx11_x (lex) } , Jump :: J11 => { lex . bump_unchecked (1usize) ; goto11_x (lex) } , } } goto20 (lex) } }
0 logos-cli/tests/data/fmt_output.rs → logos-cli/tests/data/simple/fmt_output.rs
View file
File renamed without changes.
0 logos-cli/tests/data/input.rs → logos-cli/tests/data/simple/input.rs
View file
File renamed without changes.
0 logos-cli/tests/data/output.rs → logos-cli/tests/data/simple/output.rs
View file
File renamed without changes.
111 changes: 73 additions & 38 deletions logos-cli/tests/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,80 @@ use assert_cmd::Command;
use assert_fs::{assert::PathAssert, fixture::FileWriteStr, NamedTempFile};
use predicates::prelude::*;

const INPUT_FILE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/input.rs");
const OUTPUT_FILE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/output.rs");
const FMT_OUTPUT_FILE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/fmt_output.rs");
struct Fixture {
input: &'static str,
output: &'static str,
fmt_output: &'static str,
}

const FIXTURES: &[Fixture] = &[
Fixture {
input: concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/simple/input.rs"),
output: concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/simple/output.rs"),
fmt_output: concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/data/simple/fmt_output.rs"
),
},
Fixture {
input: concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/data/no_error_lut/input.rs"
),
output: concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/data/no_error_lut/output.rs"
),
fmt_output: concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/data/no_error_lut/fmt_output.rs"
),
},
];

#[test]
fn test_codegen() {
let tempfile = NamedTempFile::new("output.gen.rs").unwrap();
for fixture in FIXTURES.iter() {
let tempfile = NamedTempFile::new("output.gen.rs").unwrap();

let mut cmd = Command::cargo_bin("logos-cli").unwrap();
cmd.arg(INPUT_FILE)
.arg("--output")
.arg(tempfile.path())
.assert()
.success();
let mut cmd = Command::cargo_bin("logos-cli").unwrap();
cmd.arg(fixture.input)
.arg("--output")
.arg(tempfile.path())
.assert()
.success();

tempfile.assert(normalize_newlines(OUTPUT_FILE));
tempfile.assert(normalize_newlines(fixture.output));
}
}

#[test]
fn test_codegen_check() {
Command::cargo_bin("logos-cli")
.unwrap()
.arg(INPUT_FILE)
.arg("--check")
.arg("--output")
.arg(OUTPUT_FILE)
.assert()
.success();
for fixture in FIXTURES.iter() {
Command::cargo_bin("logos-cli")
.unwrap()
.arg(fixture.input)
.arg("--check")
.arg("--output")
.arg(fixture.output)
.assert()
.success();
}
}

#[test]
fn test_codegen_check_format() {
Command::cargo_bin("logos-cli")
.unwrap()
.arg(INPUT_FILE)
.arg("--format")
.arg("--check")
.arg("--output")
.arg(FMT_OUTPUT_FILE)
.assert()
.success();
for fixture in FIXTURES.iter() {
Command::cargo_bin("logos-cli")
.unwrap()
.arg(fixture.input)
.arg("--format")
.arg("--check")
.arg("--output")
.arg(fixture.fmt_output)
.assert()
.success();
}
}

#[test]
Expand All @@ -55,7 +88,7 @@ fn test_codegen_fail_check() {

Command::cargo_bin("logos-cli")
.unwrap()
.arg(INPUT_FILE)
.arg(FIXTURES[0].input)
.arg("--check")
.arg("--output")
.arg(tempfile.path())
Expand All @@ -65,17 +98,19 @@ fn test_codegen_fail_check() {

#[test]
fn test_codegen_format() {
let tempfile = NamedTempFile::new("output.gen.rs").unwrap();
for fixture in FIXTURES {
let tempfile = NamedTempFile::new("output.gen.rs").unwrap();

let mut cmd = Command::cargo_bin("logos-cli").unwrap();
cmd.arg(INPUT_FILE)
.arg("--format")
.arg("--output")
.arg(tempfile.path())
.assert()
.success();
let mut cmd = Command::cargo_bin("logos-cli").unwrap();
cmd.arg(fixture.input)
.arg("--format")
.arg("--output")
.arg(tempfile.path())
.assert()
.success();

tempfile.assert(normalize_newlines(FMT_OUTPUT_FILE));
tempfile.assert(normalize_newlines(fixture.fmt_output));
}
}

fn normalize_newlines(s: impl AsRef<Path>) -> impl Predicate<str> {
Expand Down

0 comments on commit 0dfcf00

Please sign in to comment.