Skip to content
Permalink
58cf0a5005
Switch branches/tags
Go to file
 
 
Cannot retrieve contributors at this time
849 lines (721 sloc) 24.9 KB
#![cfg_attr(not(feature = "std"), no_std)]
extern crate alloc;
use alloc::collections::VecDeque;
use logos::Logos;
mod op;
mod str_block;
use op::{lex_operator, Operator};
use str_block::{lex_str_block, StringBlockToken};
pub mod unescape;
#[derive(Logos, Debug, PartialEq)]
enum RawToken {
#[token("assert")]
KeywordAssert,
#[token("else")]
KeywordElse,
#[token("error")]
KeywordError,
#[token("false")]
KeywordFalse,
#[token("for")]
KeywordFor,
#[token("function")]
KeywordFunction,
#[token("if")]
KeywordIf,
#[token("import")]
KeywordImport,
#[token("importstr")]
KeywordImportStr,
#[token("in")]
KeywordIn,
#[token("local")]
KeywordLocal,
#[token("null")]
KeywordNull,
#[token("tailstrict")]
KeywordTailStrict,
#[token("then")]
KeywordThen,
#[token("self")]
KeywordSelf,
#[token("super")]
KeywordSuper,
#[token("true")]
KeywordTrue,
#[regex(r"[_a-zA-Z][_a-zA-Z0-9]*")]
Ident,
#[regex(r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")]
Number,
#[regex(r"(?:0|[1-9][0-9]*)\.[^0-9]")]
ErrorNumJunkAfterDecimalPoint,
#[regex(r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][^+\-0-9]")]
ErrorNumJunkAfterExponent,
#[regex(r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][+-][^0-9]")]
ErrorNumJunkAfterExponentSign,
#[token("{")]
SymbolLeftBrace,
#[token("}")]
SymbolRightBrace,
#[token("[")]
SymbolLeftBracket,
#[token("]")]
SymbolRightBracket,
#[token(",")]
SymbolComma,
#[token(".")]
SymbolDot,
#[token("(")]
SymbolLeftParen,
#[token(")")]
SymbolRightParen,
#[token(";")]
SymbolSemi,
#[token("$")]
SymbolDollar,
#[regex(r"[!\$:~\+\-&\|\^=<>\*/%]+", lex_operator)]
Op(Operator),
#[regex("\"(?s:[^\"\\\\]|\\\\.)*\"")]
StringDoubleQuoted,
#[regex("'(?s:[^'\\\\]|\\\\.)*'")]
StringSingleQuoted,
#[regex("@\"(?:[^\"]|\"\")*\"")]
StringDoubleVerbatim,
#[regex("@'(?:[^']|'')*'")]
StringSingleVerbatim,
#[regex(r"\|\|\|", lex_str_block)]
StringBlock(StringBlockToken),
#[regex("\"(?s:[^\"\\\\]|\\\\.)*")]
ErrorStringDoubleQuotedUnterminated,
#[regex("'(?s:[^'\\\\]|\\\\.)*")]
ErrorStringSingleQuotedUnterminated,
#[regex("@\"(?:[^\"]|\"\")*")]
ErrorStringDoubleVerbatimUnterminated,
#[regex("@'(?:[^']|'')*")]
ErrorStringSingleVerbatimUnterminated,
#[regex("@[^\"'\\s]\\S+")]
ErrorStringMissingQuotes,
#[token("/*/")]
ErrorCommentTooShort,
#[regex(r"/\*([^*]|\*[^/])+")]
ErrorCommentUnterminated,
#[regex(r"[ \t\n\r]+")]
Whitespace,
#[regex(r"//[^\r\n]*(\r\n|\n)?")]
SingelLineSlashComment,
#[regex(r"#[^\r\n]*(\r\n|\n)?")]
SingleLineHashComment,
#[regex(r"/\*([^*]|\*[^/])*\*/")]
MultiLineComment,
#[error]
Error,
}
macro_rules! token_enum {
(
$(#[$($enum_m:tt)*])*
pub enum $name:ident {
$($case:ident $(#[$($m:tt)*])*)*
}
) => {
$(#[$($enum_m)*])*
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
pub enum $name {
$(
$(#[$($m)*])*
$case,
)*
}
};
}
token_enum! {
/// Token kind.
pub enum TokenKind {
// keywords
KeywordAssert /// `assert`
KeywordElse /// `else`
KeywordError /// `error`
KeywordFalse /// `false`
KeywordFor /// `for`
KeywordFunction /// `function`
KeywordIf /// `if`
KeywordImport /// `import`
KeywordImportStr /// `importstr`
KeywordIn /// `in`
KeywordLocal /// `local`
KeywordNull /// `null`
KeywordTailStrict /// `tailstrict`
KeywordThen /// `then`
KeywordSelf /// `self`
KeywordSuper /// `super`
KeywordTrue /// `true`
// variable tokens
Ident /// identifier token
Number /// number token
// symbols
SymbolLeftBrace /// `{`
SymbolRightBrace /// `}`
SymbolLeftBracket /// `[`
SymbolRightBracket /// `]`
SymbolComma /// `,`
SymbolDot /// `.`
SymbolLeftParen /// `(`
SymbolRightParen /// `)`
SymbolSemi /// `;`
SymbolDollar /// `$`
// operators
OpNot /// `!`
OpAssign /// `=`
OpColon /// `:`
OpDoubleColon /// `::`
OpTripleColon /// `:::`
OpPlusColon /// `+:`
OpPlusDoubleColon /// `+::`
OpPlusTripleColon /// `+:::`
OpMul /// `*`
OpDiv /// `/`
OpMod /// `%`
OpPlus /// `+`
OpMinus /// `-`
OpShiftLeft /// `<<`
OpShiftRight /// `>>`
OpLessThan /// `<`
OpGreaterThan /// `>`
OpLessThanOrEqual /// `<=`
OpGreaterThanOrEqual /// `>=`
OpEqual /// `==`
OpNotEqual /// `!=`
OpBitAnd /// `&`
OpBitXor /// `^`
OpBitOr /// `|`
OpBitNeg /// `~`
OpAnd /// `&&`
OpOr /// `||`
// strings
StringDoubleQuoted /// `"foo"`
StringSingleQuoted /// `'foo'`
StringDoubleVerbatim /// `@"foo"`
StringSingleVerbatim /// `@'foo'`
StringBlock /// ```jsonnet
/// |||
/// foo
/// |||
/// ```
Whitespace /// any whitespace
SingelLineSlashComment /// `// comment`
SingleLineHashComment /// `# comment`
BlockComment /// `/* comment */`
// string errors
ErrorStringDoubleQuotedUnterminated /// unterminated double quoted string
ErrorStringSingleQuotedUnterminated /// unterminated single quoted string
ErrorStringDoubleVerbatimUnterminated /// unterminated double quoted verbatim string
ErrorStringSingleVerbatimUnterminated /// unterminated single quoted verbatim string
ErrorStringBlockUnerminated /// unterminated block string
ErrorStringMissingQuotes /// verbatim string missing quotes
ErrorStringBlockMissingNewLine /// block string missing newline
ErrorStringBlockMissingTermination /// block string missing termination sequence
ErrorStringBlockMissingIndent /// block string missing indentation
// number errors
ErrorNumJunkAfterDecimalPoint /// unexpected character after decimal point
ErrorNumJunkAfterExponent /// unexpected character after exponent
ErrorNumJunkAfterExponentSign /// unexpected character after exponent sign
// comment errors
ErrorCommentTooShort /// multiline comment too short (`/*/`)
ErrorCommentUnterminated /// unterminated comment
// other
ErrorUnknownOperator /// unknown operator
ErrorInvalidToken /// invalid token
}
}
impl TokenKind {
pub fn is_error(self) -> bool {
use TokenKind::*;
match self {
ErrorStringDoubleQuotedUnterminated
| ErrorStringSingleQuotedUnterminated
| ErrorStringDoubleVerbatimUnterminated
| ErrorStringSingleVerbatimUnterminated
| ErrorStringBlockUnerminated
| ErrorStringMissingQuotes
| ErrorStringBlockMissingNewLine
| ErrorStringBlockMissingTermination
| ErrorStringBlockMissingIndent
| ErrorNumJunkAfterDecimalPoint
| ErrorNumJunkAfterExponent
| ErrorNumJunkAfterExponentSign
| ErrorCommentTooShort
| ErrorCommentUnterminated
| ErrorUnknownOperator
| ErrorInvalidToken => true,
_ => false,
}
}
}
/// A token of jsonnet source.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Token {
/// The kind of token.
pub kind: TokenKind,
/// The token value.
pub len: u32,
}
/// A lexer of jsonnet source.
pub struct Lexer<'a> {
inner: logos::Lexer<'a, RawToken>,
done: bool,
peeked: VecDeque<Token>,
#[cfg(debug_assertions)]
len: u32,
}
impl<'a> Lexer<'a> {
pub fn new(content: &'a str) -> Self {
Self {
inner: RawToken::lexer(content),
done: false,
peeked: VecDeque::new(),
#[cfg(debug_assertions)]
len: 0,
}
}
fn read_token(&mut self) -> Option<Token> {
if self.done {
return None;
}
match self.inner.next() {
None => {
self.done = true;
None
}
Some(raw) => {
let len = self.inner.slice().len() as u32;
#[cfg(debug_assertions)]
{
self.len += len;
assert_eq!(self.len, self.inner.span().end as u32);
}
Some(self.to_token(raw, len))
}
}
}
fn to_token(&self, raw: RawToken, len: u32) -> Token {
let kind = match raw {
RawToken::KeywordAssert => TokenKind::KeywordAssert,
RawToken::KeywordElse => TokenKind::KeywordElse,
RawToken::KeywordError => TokenKind::KeywordError,
RawToken::KeywordFalse => TokenKind::KeywordFalse,
RawToken::KeywordFor => TokenKind::KeywordFor,
RawToken::KeywordFunction => TokenKind::KeywordFunction,
RawToken::KeywordIf => TokenKind::KeywordIf,
RawToken::KeywordImport => TokenKind::KeywordImport,
RawToken::KeywordImportStr => TokenKind::KeywordImportStr,
RawToken::KeywordIn => TokenKind::KeywordIn,
RawToken::KeywordLocal => TokenKind::KeywordLocal,
RawToken::KeywordNull => TokenKind::KeywordNull,
RawToken::KeywordTailStrict => TokenKind::KeywordTailStrict,
RawToken::KeywordThen => TokenKind::KeywordThen,
RawToken::KeywordSelf => TokenKind::KeywordSelf,
RawToken::KeywordSuper => TokenKind::KeywordSuper,
RawToken::KeywordTrue => TokenKind::KeywordTrue,
RawToken::Ident => TokenKind::Ident,
RawToken::Number => TokenKind::Number,
RawToken::SymbolLeftBrace => TokenKind::SymbolLeftBrace,
RawToken::SymbolRightBrace => TokenKind::SymbolRightBrace,
RawToken::SymbolLeftBracket => TokenKind::SymbolLeftBracket,
RawToken::SymbolRightBracket => TokenKind::SymbolRightBracket,
RawToken::SymbolComma => TokenKind::SymbolComma,
RawToken::SymbolDot => TokenKind::SymbolDot,
RawToken::SymbolLeftParen => TokenKind::SymbolLeftParen,
RawToken::SymbolRightParen => TokenKind::SymbolRightParen,
RawToken::SymbolSemi => TokenKind::SymbolSemi,
RawToken::SymbolDollar => TokenKind::SymbolDollar,
RawToken::Op(Operator::Not) => TokenKind::OpNot,
RawToken::Op(Operator::Assign) => TokenKind::OpAssign,
RawToken::Op(Operator::Colon) => TokenKind::OpColon,
RawToken::Op(Operator::DoubleColon) => TokenKind::OpDoubleColon,
RawToken::Op(Operator::TripleColon) => TokenKind::OpTripleColon,
RawToken::Op(Operator::PlusColon) => TokenKind::OpPlusColon,
RawToken::Op(Operator::PlusDoubleColon) => TokenKind::OpPlusDoubleColon,
RawToken::Op(Operator::PlusTripleColon) => TokenKind::OpPlusTripleColon,
RawToken::Op(Operator::Mul) => TokenKind::OpMul,
RawToken::Op(Operator::Div) => TokenKind::OpDiv,
RawToken::Op(Operator::Mod) => TokenKind::OpMod,
RawToken::Op(Operator::Plus) => TokenKind::OpPlus,
RawToken::Op(Operator::Minus) => TokenKind::OpMinus,
RawToken::Op(Operator::ShiftLeft) => TokenKind::OpShiftLeft,
RawToken::Op(Operator::ShiftRight) => TokenKind::OpShiftRight,
RawToken::Op(Operator::LessThan) => TokenKind::OpLessThan,
RawToken::Op(Operator::GreaterThan) => TokenKind::OpGreaterThan,
RawToken::Op(Operator::LessThanOrEqual) => TokenKind::OpLessThanOrEqual,
RawToken::Op(Operator::GreaterThanOrEqual) => TokenKind::OpGreaterThanOrEqual,
RawToken::Op(Operator::Equal) => TokenKind::OpEqual,
RawToken::Op(Operator::NotEqual) => TokenKind::OpNotEqual,
RawToken::Op(Operator::BitAnd) => TokenKind::OpBitAnd,
RawToken::Op(Operator::BitXor) => TokenKind::OpBitXor,
RawToken::Op(Operator::BitOr) => TokenKind::OpBitOr,
RawToken::Op(Operator::BitNeg) => TokenKind::OpBitNeg,
RawToken::Op(Operator::And) => TokenKind::OpAnd,
RawToken::Op(Operator::Or) => TokenKind::OpOr,
RawToken::StringDoubleQuoted => TokenKind::StringDoubleQuoted,
RawToken::StringSingleQuoted => TokenKind::StringSingleQuoted,
RawToken::StringDoubleVerbatim => TokenKind::StringDoubleVerbatim,
RawToken::StringSingleVerbatim => TokenKind::StringSingleVerbatim,
RawToken::StringBlock(StringBlockToken::Valid) => TokenKind::StringBlock,
RawToken::Whitespace => TokenKind::Whitespace,
RawToken::SingelLineSlashComment => TokenKind::SingelLineSlashComment,
RawToken::SingleLineHashComment => TokenKind::SingleLineHashComment,
RawToken::MultiLineComment => TokenKind::BlockComment,
RawToken::ErrorNumJunkAfterDecimalPoint => TokenKind::ErrorNumJunkAfterDecimalPoint,
RawToken::ErrorNumJunkAfterExponent => TokenKind::ErrorNumJunkAfterExponent,
RawToken::ErrorNumJunkAfterExponentSign => TokenKind::ErrorNumJunkAfterExponentSign,
RawToken::ErrorStringDoubleQuotedUnterminated => {
TokenKind::ErrorStringDoubleQuotedUnterminated
}
RawToken::ErrorStringSingleQuotedUnterminated => {
TokenKind::ErrorStringSingleQuotedUnterminated
}
RawToken::ErrorStringDoubleVerbatimUnterminated => {
TokenKind::ErrorStringDoubleVerbatimUnterminated
}
RawToken::ErrorStringSingleVerbatimUnterminated => {
TokenKind::ErrorStringSingleVerbatimUnterminated
}
RawToken::ErrorStringMissingQuotes => TokenKind::ErrorStringMissingQuotes,
RawToken::StringBlock(StringBlockToken::UnexpectedEndOfString) => {
TokenKind::ErrorStringBlockUnerminated
}
RawToken::StringBlock(StringBlockToken::MissingTextBlockIndent) => {
TokenKind::ErrorStringBlockMissingIndent
}
RawToken::StringBlock(StringBlockToken::MissingTextBlockNewLine) => {
TokenKind::ErrorStringBlockMissingNewLine
}
RawToken::StringBlock(StringBlockToken::MissingTextBlockTermination) => {
TokenKind::ErrorStringBlockMissingTermination
}
RawToken::ErrorCommentTooShort => TokenKind::ErrorCommentTooShort,
RawToken::ErrorCommentUnterminated => TokenKind::ErrorCommentUnterminated,
RawToken::Op(Operator::Unknown) => TokenKind::ErrorUnknownOperator,
RawToken::Error => TokenKind::ErrorInvalidToken,
};
Token { kind, len }
}
#[inline]
pub fn peek(&mut self) -> Option<Token> {
self.peek_nth(0)
}
pub fn peek_nth(&mut self, n: usize) -> Option<Token> {
while self.peeked.len() <= n && !self.done {
if let Some(tok) = self.read_token() {
self.peeked.push_back(tok);
}
}
self.peeked.get(n).copied()
}
}
impl<'a> Iterator for Lexer<'a> {
type Item = Token;
#[inline]
fn next(&mut self) -> Option<Self::Item> {
match self.peeked.pop_front() {
Some(v) => Some(v),
None => self.read_token(),
}
}
}
/// Tokenize a string of jsonnet into a list of tokens.
pub fn tokenize<'a>(content: &'a str) -> impl Iterator<Item = Token> + 'a {
Lexer::new(content)
}
#[cfg(test)]
mod tests {
use super::{TokenKind::*, *};
use test_case::test_case;
macro_rules! test_tokens {
($src:expr, [$(
$tok:expr
$(=> $val:expr)?
),*$(,)?]) => {#[allow(unused)] {
let src: &str = $src;
let mut lex = Lexer::new(src);
let mut index = 0;
let mut offset = 0;
$({
let actual = lex.next().expect(&format!("Expected token {}", index + 1));
let expected = $tok;
assert_eq!(actual.kind, expected, "index: {}", index);
$(
let val = &src[offset as usize..(offset + actual.len) as usize];
assert_eq!(val, $val, "index: {}", index);
)?
index += 1;
offset += actual.len;
})*
match lex.next() {
None => (),
Some(t) => panic!("Expected exactly {} tokens, but got {:#?} when expecting EOF", index, t),
}
}};
}
#[test]
fn empty() {
test_tokens!("", []);
}
#[test]
fn whitespace() {
test_tokens!(" \t\n\r\r\n", [Whitespace]);
}
#[test_case("{", SymbolLeftBrace)]
#[test_case("}", SymbolRightBrace)]
#[test_case("[", SymbolLeftBracket)]
#[test_case("]", SymbolRightBracket)]
#[test_case("(", SymbolLeftParen)]
#[test_case(")", SymbolRightParen)]
#[test_case(",", SymbolComma)]
#[test_case(".", SymbolDot)]
#[test_case(";", SymbolSemi)]
#[test_case("$", SymbolDollar)]
fn symbol(src: &str, tok: TokenKind) {
test_tokens!(src, [tok]);
}
#[test_case(":", OpColon)]
#[test_case("::", OpDoubleColon)]
#[test_case("!", OpNot)]
#[test_case("==", OpEqual)]
#[test_case("!=", OpNotEqual)]
#[test_case("~", OpBitNeg)]
#[test_case("+", OpPlus)]
#[test_case("-", OpMinus)]
#[test_case("*", OpMul)]
#[test_case("/", OpDiv)]
#[test_case("%", OpMod)]
#[test_case("&", OpBitAnd)]
#[test_case("|", OpBitOr)]
#[test_case("^", OpBitXor)]
#[test_case("=", OpAssign)]
#[test_case("<", OpLessThan)]
#[test_case(">", OpGreaterThan)]
#[test_case("<=", OpLessThanOrEqual)]
#[test_case(">=", OpGreaterThanOrEqual)]
fn operator(src: &str, tok: TokenKind) {
test_tokens!(src, [tok]);
}
#[test_case("->" ; "arrow_right")]
#[test_case("<-" ; "arrow_left")]
#[test_case(">==|" ; "junk")]
fn bad_op(src: &str) {
test_tokens!(src, [ErrorUnknownOperator]);
}
#[test_case("1")]
#[test_case("1.0")]
#[test_case("0.10")]
#[test_case("0e100")]
#[test_case("1e100")]
#[test_case("1.1e100")]
#[test_case("1.2e-100")]
#[test_case("1.3e+100")]
fn number(src: &str) {
test_tokens!(src, [Number]);
}
#[test]
fn number_0100() {
test_tokens!("0100", [Number=>"0", Number=>"100",]);
}
#[test]
fn number_10_p_10() {
test_tokens!(
"10+11",
[
Number=>"10",
OpPlus,
Number=>"11",
]
);
}
#[test_case("1.+", ErrorNumJunkAfterDecimalPoint)]
#[test_case("1e!", ErrorNumJunkAfterExponent)]
#[test_case("1e+!", ErrorNumJunkAfterExponentSign)]
fn bad_number(src: &str, tok: TokenKind) {
test_tokens!(src, [tok]);
}
#[test_case("\"hi\"", StringDoubleQuoted ; "double_1")]
#[test_case("\"hi\n\"", StringDoubleQuoted ; "double_2")]
#[test_case("\"hi\\\"\"", StringDoubleQuoted ; "double_3")]
#[test_case("'hi'", StringSingleQuoted ; "single_1")]
#[test_case("'hi\n'", StringSingleQuoted ; "single_2")]
#[test_case("'hi\\''", StringSingleQuoted ; "single_3")]
#[test_case("|||\n test\n more\n |||\n foo\n|||", StringBlock ; "block_1")]
#[test_case("|||\n\ttest\n\t more\n\t|||\n\t foo\n|||", StringBlock ; "block_2")]
#[test_case("|||\n\t \ttest\n\t \t more\n\t \t|||\n\t \t foo\n|||", StringBlock ; "block_3")]
#[test_case("|||\n\n test\n\n\n more\n |||\n foo\n|||", StringBlock ; "block_4")]
#[test_case("@\"\"", StringDoubleVerbatim ; "verbatim_double_1")]
#[test_case("@''", StringSingleVerbatim ; "verbatim_single_1")]
#[test_case("@\"\"\"\"", StringDoubleVerbatim ; "verbatim_double_2")]
#[test_case("@''''", StringSingleVerbatim ; "verbatim_single_2")]
#[test_case("@\"\\n\"", StringDoubleVerbatim ; "verbatim_double_3")]
#[test_case("@\"''\"", StringDoubleVerbatim ; "verbatim_double_4")]
fn string(src: &str, tok: TokenKind) {
test_tokens!(src, [tok]);
}
#[test_case("\"hi", ErrorStringDoubleQuotedUnterminated ; "double_unterm")]
#[test_case("'hi", ErrorStringSingleQuotedUnterminated ; "single_unterm")]
#[test_case("@\"hi", ErrorStringDoubleVerbatimUnterminated ; "double_verb_unterm")]
#[test_case("@'hi", ErrorStringSingleVerbatimUnterminated ; "single_verb_unterm")]
fn string_unterminated(src: &str, tok: TokenKind) {
test_tokens!(src, [tok]);
}
#[test_case("assert", KeywordAssert)]
#[test_case("else", KeywordElse)]
#[test_case("error", KeywordError)]
#[test_case("false", KeywordFalse)]
#[test_case("for", KeywordFor)]
#[test_case("function", KeywordFunction)]
#[test_case("if", KeywordIf)]
#[test_case("import", KeywordImport)]
#[test_case("importstr", KeywordImportStr)]
#[test_case("in", KeywordIn)]
#[test_case("local", KeywordLocal)]
#[test_case("null", KeywordNull)]
#[test_case("self", KeywordSelf)]
#[test_case("super", KeywordSuper)]
#[test_case("tailstrict", KeywordTailStrict)]
#[test_case("then", KeywordThen)]
#[test_case("true", KeywordTrue)]
fn keyword(src: &str, tok: TokenKind) {
test_tokens!(src, [tok]);
}
#[test]
fn identifier() {
test_tokens!("foobar123", [Ident=>"foobar123"]);
}
#[test]
fn identifiers() {
test_tokens!(
"foo bar123",
[
Ident=>"foo",
Whitespace,
Ident=>"bar123",
]
);
}
#[test]
fn c_comment() {
test_tokens!("// hi", [SingelLineSlashComment]);
}
#[test]
fn py_comment() {
test_tokens!("# hi", [SingleLineHashComment]);
}
#[test]
fn c_multiline_comment() {
test_tokens!("/* hi \n bye */", [BlockComment]);
}
#[test]
fn c_comment_too_short() {
test_tokens!("/*/", [ErrorCommentTooShort]);
}
#[test]
fn c_comment_minimal() {
test_tokens!("/**/", [BlockComment]);
}
#[test]
fn c_comment_just_slack() {
test_tokens!("/*/*/", [BlockComment]);
}
#[test]
fn c_comment_space_slack() {
test_tokens!("/* /*/", [BlockComment]);
}
#[test]
fn c_comment_many_lines() {
test_tokens!("/*\n\n*/", [BlockComment]);
}
#[test]
fn c_comment_no_term() {
test_tokens!("/* hi", [ErrorCommentUnterminated]);
}
// TODO: Figure out
// #[test]
// fn str_block_bad_indent() {
// test_tokens!("|||\n test\n foo\n|||", [ErrorStringBlockMissingIndent]);
// }
#[test]
fn str_block_eof() {
test_tokens!("|||\n test", [ErrorStringBlockUnerminated]);
}
#[test]
fn str_block_not_term() {
test_tokens!("|||\n test\n", [ErrorStringBlockUnerminated]);
}
#[test]
fn str_block_no_ws() {
test_tokens!("|||\ntest\n|||", [ErrorStringBlockMissingIndent]);
}
#[test]
fn str_verbatim_unterminated() {
test_tokens!("@\"blah blah", [ErrorStringDoubleVerbatimUnterminated]);
}
#[test]
fn str_verbatim_junk_after_at() {
test_tokens!(
"@blah blah",
[
ErrorStringMissingQuotes,
Whitespace,
Ident=>"blah",
]
);
}
#[test]
fn junk() {
let src = "💩";
test_tokens!(src, [ErrorInvalidToken]);
}
mod golden {
use super::*;
use core::fmt;
#[derive(PartialEq, Eq, Clone, Copy)]
struct PrettyString<'a>(&'a str);
impl<'a> PrettyString<'a> {
#[inline]
fn new<S: AsRef<str> + ?Sized>(value: &'a S) -> Self {
PrettyString(value.as_ref())
}
}
impl<'a> fmt::Debug for PrettyString<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(self.0)
}
}
macro_rules! assert_eq {
($left:expr, $right:expr $(,$($rest:tt)*)?) => {
pretty_assertions::assert_eq!(
PrettyString::new($left),
PrettyString::new($right)
$(,$($rest)*)?
);
}
}
#[test_gen::test_golden("test_data/inline/ok/*.jsonnet")]
fn golden_ok(content: &str, _: &str) -> String {
use core::fmt::Write;
let lex = Lexer::new(content);
let mut result = String::new();
let mut offset = 0;
for token in lex {
let val = &content[offset as usize..(offset + token.len) as usize];
assert!(!token.kind.is_error());
writeln!(
result,
"{:?}@{}:{} {:?}",
token.kind,
offset,
offset + token.len,
val
)
.unwrap();
offset += token.len;
}
result
}
}
}