Skip to content

Commit

Permalink
fix(parser): Don't stop on almost-printfs
Browse files Browse the repository at this point in the history 
When we added support for printf interopolation, we had to adjust our
separator matching to not eat the start of printf interpolation.

When doing so, I overlooked the need to still eat it in the catch-all.
If we don't, we then try to read `%` as part of the identifier and bail
out early.

Fixes crate-ci#411
  • Loading branch information
@epage
epage committed Jan 26, 2022
1 parent 4b2e664 commit 3c78d65
Showing 1 changed file with 59 additions and 18 deletions.
77 changes: 59 additions & 18 deletions crates/typos/src/tokens.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ mod parser {

fn identifier<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition,
T: nom::InputTakeAtPosition + std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
{
// Generally a language would be `{XID_Start}{XID_Continue}*` but going with only
Expand Down Expand Up @@ -191,18 +191,39 @@ mod parser {
terminated(url_literal, sep1),
c_escape,
printf,
sep1,
other,
)))(input)
}

fn sep1<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition,
T: nom::InputTakeAtPosition + std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
{
take_while1(is_ignore_char)(input)
}

fn other<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition
+ nom::InputTake
+ nom::InputIter
+ nom::InputLength
+ nom::Slice<std::ops::RangeFrom<usize>>
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Offset
+ Clone
+ PartialEq
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
recognize(tuple((
satisfy(|c| !is_xid_continue(c)),
take_while(is_ignore_char),
)))(input)
}

fn ordinal_literal<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition
Expand All @@ -212,7 +233,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand All @@ -237,7 +259,7 @@ mod parser {

fn dec_literal<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition,
T: nom::InputTakeAtPosition + std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
{
take_while1(is_dec_digit_with_sep)(input)
Expand All @@ -250,7 +272,8 @@ mod parser {
+ nom::InputIter
+ nom::InputLength
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand All @@ -269,7 +292,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand All @@ -295,7 +319,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand Down Expand Up @@ -325,8 +350,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand Down Expand Up @@ -359,8 +384,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand All @@ -380,8 +405,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand Down Expand Up @@ -412,8 +437,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand All @@ -433,8 +458,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
Expand Down Expand Up @@ -1159,6 +1184,22 @@ mod test {
assert_eq!(expected, actual);
}

#[test]
fn tokenize_template() {
let parser = TokenizerBuilder::new().build();

let input = "Hello {{% foo %}} world!";
let expected: Vec<Identifier> = vec![
Identifier::new_unchecked("Hello", Case::None, 0),
Identifier::new_unchecked("foo", Case::None, 10),
Identifier::new_unchecked("world", Case::None, 18),
];
let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect();
assert_eq!(expected, actual);
let actual: Vec<_> = parser.parse_str(input).collect();
assert_eq!(expected, actual);
}

#[test]
fn split_ident() {
let cases = [
Expand Down

0 comments on commit 3c78d65

Please sign in to comment.