diff --git a/src/ast.rs b/src/ast.rs index 9c0affe..e930749 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -68,10 +68,31 @@ impl<'a> From> for serde_json::Value { } Value::BooleanLit(b) => serde_json::Value::Bool(b.value), Value::NullKeyword(_) => serde_json::Value::Null, - Value::NumberLit(num) => match serde_json::Number::from_str(num.value) { - Ok(number) => serde_json::Value::Number(number), - Err(_) => serde_json::Value::String(num.value.to_string()), - }, + Value::NumberLit(num) => { + // Check if this is a hexadecimal literal (0x or 0X prefix) + let num_str = num.value.trim_start_matches('-'); + if num_str.len() > 2 && (num_str.starts_with("0x") || num_str.starts_with("0X")) { + // Parse hexadecimal and convert to decimal + let hex_part = &num_str[2..]; + match i64::from_str_radix(hex_part, 16) { + Ok(decimal_value) => { + let final_value = if num.value.starts_with('-') { + -decimal_value + } else { + decimal_value + }; + serde_json::Value::Number(serde_json::Number::from(final_value)) + } + Err(_) => serde_json::Value::String(num.value.to_string()), + } + } else { + // Standard decimal number + match serde_json::Number::from_str(num.value) { + Ok(number) => serde_json::Value::Number(number), + Err(_) => serde_json::Value::String(num.value.to_string()), + } + } + } Value::Object(obj) => { let mut map = serde_json::map::Map::new(); for prop in obj.properties { diff --git a/src/cst/mod.rs b/src/cst/mod.rs index 367ee26..dbd7f9b 100644 --- a/src/cst/mod.rs +++ b/src/cst/mod.rs @@ -1442,10 +1442,30 @@ impl CstNumberLit { pub fn to_serde_value(&self) -> Option { use std::str::FromStr; let raw = self.0.borrow().value.clone(); - match serde_json::Number::from_str(&raw) { - Ok(number) => Some(serde_json::Value::Number(number)), - // If the number is invalid, return it as a string (same behavior as AST conversion) - Err(_) => Some(serde_json::Value::String(raw)), + + // check if this is a hexadecimal literal (0x or 0X prefix) + let num_str = raw.trim_start_matches('-'); + if num_str.len() > 2 && (num_str.starts_with("0x") || num_str.starts_with("0X")) { + // parse hexadecimal and convert to decimal + let hex_part = &num_str[2..]; + match i64::from_str_radix(hex_part, 16) { + Ok(decimal_value) => { + let final_value = if raw.starts_with('-') { + -decimal_value + } else { + decimal_value + }; + Some(serde_json::Value::Number(serde_json::Number::from(final_value))) + } + Err(_) => Some(serde_json::Value::String(raw)), + } + } else { + // standard decimal number + match serde_json::Number::from_str(&raw) { + Ok(number) => Some(serde_json::Value::Number(number)), + // if the number is invalid, return it as a string (same behavior as AST conversion) + Err(_) => Some(serde_json::Value::String(raw)), + } } } } diff --git a/src/parse_to_ast.rs b/src/parse_to_ast.rs index fee68cb..1534a43 100644 --- a/src/parse_to_ast.rs +++ b/src/parse_to_ast.rs @@ -573,4 +573,33 @@ mod tests { fn error_correct_line_column_unicode_width() { assert_has_strict_error(r#"["🧑‍🦰", ["#, "Unterminated array on line 1 column 10"); } + + #[test] + fn it_should_parse_unquoted_keys_with_hex_and_trailing_comma() { + let text = r#"{ + CP_CanFuncReqId: 0x7DF, // 2015 + }"#; + { + let parse_result = parse_to_ast(text, &Default::default(), &Default::default()).unwrap(); + + let value = parse_result.value.unwrap(); + let obj = value.as_object().unwrap(); + assert_eq!(obj.properties.len(), 1); + assert_eq!(obj.properties[0].name.as_str(), "CP_CanFuncReqId"); + + let number_value = obj.properties[0].value.as_number_lit().unwrap(); + assert_eq!(number_value.value, "0x7DF"); + } + #[cfg(feature = "serde")] + { + let value = crate::parse_to_serde_value(text, &Default::default()).unwrap().unwrap(); + // hexadecimal numbers are converted to decimal in serde output + assert_eq!( + value, + serde_json::json!({ + "CP_CanFuncReqId": 2015 + }) + ); + } + } } diff --git a/src/scanner.rs b/src/scanner.rs index de9695b..573f13c 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -160,6 +160,23 @@ impl<'a> Scanner<'a> { if self.is_zero() { self.move_next_char(); + + // check for hexadecimal literal (0x or 0X) + if matches!(self.current_char(), Some('x') | Some('X')) { + self.move_next_char(); + + // must have at least one hex digit + if !self.is_hex_digit() { + return Err(self.create_error_for_current_char(ParseErrorKind::ExpectedDigit)); + } + + while self.is_hex_digit() { + self.move_next_char(); + } + + let end_byte_index = self.byte_index; + return Ok(Token::Number(&self.file_text[start_byte_index..end_byte_index])); + } } else if self.is_one_nine() { self.move_next_char(); while self.is_digit() { @@ -288,10 +305,12 @@ impl<'a> Scanner<'a> { let start_byte_index = self.byte_index; while let Some(current_char) = self.current_char() { - if current_char.is_whitespace() || current_char == '\r' || current_char == '\n' || current_char == ':' { + // check for word terminators + if current_char.is_whitespace() || current_char == ':' { break; } - if !current_char.is_alphanumeric() && current_char != '-' { + // validate that the character is allowed in a word literal + if !current_char.is_alphanumeric() && current_char != '-' && current_char != '_' { return Err(self.create_error_for_current_token(ParseErrorKind::UnexpectedToken)); } @@ -382,6 +401,13 @@ impl<'a> Scanner<'a> { self.is_one_nine() || self.is_zero() } + fn is_hex_digit(&self) -> bool { + match self.current_char() { + Some(current_char) => current_char.is_ascii_hexdigit(), + _ => false, + } + } + fn is_zero(&self) -> bool { self.current_char() == Some('0') } @@ -496,6 +522,24 @@ mod tests { ); } + #[test] + fn it_tokenizes_hexadecimal_numbers() { + assert_has_tokens( + "0x7DF, 0xFF, 0x123ABC, 0xabc, 0X1F", + vec![ + Token::Number("0x7DF"), + Token::Comma, + Token::Number("0xFF"), + Token::Comma, + Token::Number("0x123ABC"), + Token::Comma, + Token::Number("0xabc"), + Token::Comma, + Token::Number("0X1F"), + ], + ); + } + #[test] fn it_errors_invalid_exponent() { assert_has_error( diff --git a/src/serde.rs b/src/serde.rs index 85816ee..8a57497 100644 --- a/src/serde.rs +++ b/src/serde.rs @@ -84,4 +84,24 @@ mod tests { assert_eq!(result, Some(SerdeValue::Object(expected_value))); } + + #[test] + fn it_should_parse_hexadecimal_numbers_to_decimal() { + let result = parse_to_serde_value( + r#"{ + "hex1": 0x7DF, + "hex2": 0xFF, + "hex3": 0x10 + }"#, + &Default::default(), + ) + .unwrap(); + + let mut expected_value = serde_json::map::Map::new(); + expected_value.insert("hex1".to_string(), SerdeValue::Number(serde_json::Number::from(2015))); + expected_value.insert("hex2".to_string(), SerdeValue::Number(serde_json::Number::from(255))); + expected_value.insert("hex3".to_string(), SerdeValue::Number(serde_json::Number::from(16))); + + assert_eq!(result, Some(SerdeValue::Object(expected_value))); + } }