Skip to content

Commit

Permalink
fix exponential number parsing (#8)
Browse files Browse the repository at this point in the history
  • Loading branch information
@deankarn
deankarn committed Jul 20, 2022
1 parent 0be00b6 commit 8b3a382
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 10 deletions.
9 changes: 7 additions & 2 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.5.0] - 2022-07-19
## [0.6.1] - 2022-07-19
### Fixed
- Fixed number parsing for exponential numbers eg. 1e10.

## [0.6.0] - 2022-07-19
### Added
- Added BETWEEN operator support <value> BETWEEN <value> <value>

Expand Down Expand Up @@ -69,7 +73,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Initial release.

[Unreleased]: https://github.com/rust-playground/ksql/compare/v0.6.0...HEAD
[Unreleased]: https://github.com/rust-playground/ksql/compare/v0.6.1...HEAD
[0.6.1]: https://github.com/rust-playground/ksql/compare/v0.6.0...v0.6.1
[0.6.0]: https://github.com/rust-playground/ksql/compare/v0.5.0...v0.6.0
[0.5.0]: https://github.com/rust-playground/ksql/compare/v0.4.1...v0.5.0
[0.4.1]: https://github.com/rust-playground/ksql/compare/v0.4.0...v0.4.1
Expand Down
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
name = "ksql"
description = "A JSON data expression lexer, parser, cli and library"
version = "0.6.0"
version = "0.6.1"
edition = "2021"
license = "MIT OR Apache-2.0"
readme = "README.md"
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ Expressions support most mathematical and string expressions see below for detai
| `CloseBracket` | `]` | N/A |
| `Comma` | `,` | N/A |
| `QuotedString` | `"sample text"` | Must start and end with an unescaped `"` character |
| `Number` | `123.45` | Must start and end with a valid `0-9` digit. |
| `Number` | ` 123.45 ` | Must start and end with a space or '+' or '-' when hard coded value in expression and supports `0-9 +- e` characters for numbers and exponent notation. |
| `BooleanTrue` | `true` | Accepts `true` as a boolean only. |
| `BooleanFalse` | `false` | Accepts `false` as a boolean only. |
| `SelectorPath` | `.selector_path` | Starts with a `.` and ends with whitespace blank space. This crate currently uses [gjson](https://github.com/tidwall/gjson.rs) and so the full gjson syntax for identifiers is supported. |
Expand Down
60 changes: 54 additions & 6 deletions src/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
//! | `CloseBracket` | `]` | N/A |
//! | `Comma` | `,` | N/A |
//! | `QuotedString` | `"sample text"` | Must start and end with an unescaped `"` character |
//! | `Number` | `123.45` | Must start and end with a valid `0-9` digit. |
//! | `Number` | ` 123.45 ` | Must start and end with a space or '+' or '-' when hard coded value in expression and supports `0-9 +- e` characters for numbers and exponent notation. |
//! | `BooleanTrue` | `true` | Accepts `true` as a boolean only. |
//! | `BooleanFalse` | `false` | Accepts `false` as a boolean only. |
//! | `SelectorPath` | `.selector_path` | Starts with a `.` and ends with whitespace blank space. This crate currently uses [gjson](https://github.com/tidwall/gjson.rs) and so the full gjson syntax for identifiers is supported. |
Expand Down Expand Up @@ -203,8 +203,20 @@ fn tokenize_single_token(data: &[u8]) -> Result<(TokenKind, u16)> {
let (token, end) = match b {
b'=' if data.get(1) == Some(&b'=') => (TokenKind::Equals, 2),
b'=' => (TokenKind::Equals, 1),
b'+' => (TokenKind::Add, 1),
b'-' => (TokenKind::Subtract, 1),
b'+' => {
if data.get(1).map_or_else(|| false, u8::is_ascii_digit) {
tokenize_number(data)?
} else {
(TokenKind::Add, 1)
}
}
b'-' => {
if data.get(1).map_or_else(|| false, u8::is_ascii_digit) {
tokenize_number(data)?
} else {
(TokenKind::Subtract, 1)
}
}
b'*' => (TokenKind::Multiply, 1),
b'/' => (TokenKind::Divide, 1),
b'>' if data.get(1) == Some(&b'=') => (TokenKind::Gte, 2),
Expand Down Expand Up @@ -244,8 +256,8 @@ fn tokenize_single_token(data: &[u8]) -> Result<(TokenKind, u16)> {
b'E' => tokenize_keyword(data, "ENDSWITH".as_bytes(), TokenKind::EndsWith)?,
b'B' => tokenize_keyword(data, "BETWEEN".as_bytes(), TokenKind::Between)?,
b'N' => tokenize_null(data)?,
c if c.is_ascii_digit() => tokenize_number(data)?,
b'_' => tokenize_identifier(data)?,
b'0'..=b'9' => tokenize_number(data)?,
_ => return Err(Error::UnsupportedCharacter(*b)),
};
Ok((token, end))
Expand Down Expand Up @@ -375,8 +387,8 @@ fn tokenize_number(data: &[u8]) -> Result<(TokenKind, u16)> {
true
}
}
b'-' | b'+' => true,
_ => c.is_ascii_alphanumeric(),
b'-' | b'+' | b'e' => true,
_ => c.is_ascii_digit(),
}) {
Some(end) if !bad_number => Ok((TokenKind::Number, end)),
_ => Err(Error::InvalidNumber(
Expand Down Expand Up @@ -894,4 +906,40 @@ mod tests {
" BETWEEN",
Error::InvalidKeyword("BETWEEN".to_string())
);
lex_test!(
parse_negative_number,
" -1.23 ",
Token {
kind: TokenKind::Number,
start: 1,
len: 5
}
);
lex_test!(
parse_positive_number,
" +1.23 ",
Token {
kind: TokenKind::Number,
start: 1,
len: 5
}
);
lex_test!(
parse_negative_exponent_number,
" -1e10 ",
Token {
kind: TokenKind::Number,
start: 1,
len: 5
}
);
lex_test!(
parse_plus_exponent_number,
" +1e10 ",
Token {
kind: TokenKind::Number,
start: 1,
len: 5
}
);
}
20 changes: 20 additions & 0 deletions src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1623,4 +1623,24 @@ mod tests {

Ok(())
}

#[test]
fn parse_exponent_number() -> anyhow::Result<()> {
let expression = "1e3 == 1000";
let ex = Parser::parse(expression)?;
let result = ex.calculate("".as_bytes())?;
assert_eq!(Value::Bool(true), result);

let expression = "-1e-3 == -0.001";
let ex = Parser::parse(expression)?;
let result = ex.calculate("".as_bytes())?;
assert_eq!(Value::Bool(true), result);

let expression = "+1e-3 == 0.001";
let ex = Parser::parse(expression)?;
let result = ex.calculate("".as_bytes())?;
assert_eq!(Value::Bool(true), result);

Ok(())
}
}

0 comments on commit 8b3a382

Please sign in to comment.