diff --git a/src/parser.rs b/src/parser.rs index 46937be..019fe5a 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -433,6 +433,11 @@ impl<'a, const GUARANTEED_UTF8: bool> Tokenizer<'a, GUARANTEED_UTF8> { Ok(()) } + /// Peeks at the next non-whitespace character, and returns a what kind of + /// token the next token will be if the input is valid JSON syntax. + /// + /// If the source input has no remaining non-whitespace characters, this + /// function returns `None`. #[inline] pub fn peek(&mut self) -> Option { self.source.skip_ws(); @@ -447,7 +452,7 @@ impl<'a, const GUARANTEED_UTF8: bool> Tokenizer<'a, GUARANTEED_UTF8> { b',' => PeekableTokenKind::Comma, b':' => PeekableTokenKind::Colon, b'"' => PeekableTokenKind::String, - b'-' | b'0' | b'1'..=b'9' => PeekableTokenKind::Number, + b'-' | b'0'..=b'9' => PeekableTokenKind::Number, b't' => PeekableTokenKind::True, b'f' => PeekableTokenKind::False, b'n' => PeekableTokenKind::Null, @@ -1277,3 +1282,34 @@ fn tokenizes() { &[Token::Bool(false)] ); } + +#[test] +fn tokenizer_peek_tests() { + let mut tokenizer = Tokenizer::for_json(r#"{"a": [1, true, false, null]} "#); + + for expected in [ + PeekableTokenKind::Object, + PeekableTokenKind::String, + PeekableTokenKind::Colon, + PeekableTokenKind::Array, + PeekableTokenKind::Number, + PeekableTokenKind::Comma, + PeekableTokenKind::True, + PeekableTokenKind::Comma, + PeekableTokenKind::False, + PeekableTokenKind::Comma, + PeekableTokenKind::Null, + PeekableTokenKind::ArrayEnd, + PeekableTokenKind::ObjectEnd, + ] { + assert_eq!(tokenizer.peek(), Some(expected)); + tokenizer.next().expect("eof").expect("invalid tokens"); + } + assert_eq!(tokenizer.peek(), None); + assert_eq!(tokenizer.next(), None); + + assert_eq!( + Tokenizer::for_json("<").peek(), + Some(PeekableTokenKind::Unrecognized) + ); +}