Skip to content

Validate JSON scalars in place instead of materializing byte lists #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ESRogs wants to merge 1 commit into from
Closed

Validate JSON scalars in place instead of materializing byte lists #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 19 additions & 22 deletions src/build/roc/Builtin.roc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,9 +411,7 @@ Builtin :: [].{

json_dec_digits_are_zero : Str -> Bool
json_dec_digits_are_zero = |digits| {
bytes = Str.to_utf8(digits)

for byte in bytes {
for byte in Str.iter_utf8(digits) {
if byte != 48 {
return False
}
Expand All @@ -424,16 +422,16 @@ Builtin :: [].{

trim_json_dec_leading_zeros : Str, I64 -> { digits : Str, digits_len : U64, point : I64 }
trim_json_dec_leading_zeros = |digits, point| {
bytes = Str.to_utf8(digits)
len = List.len(bytes)
len = Str.count_utf8_bytes(digits)
var $index = 0

while $index < len and list_get_unsafe(bytes, $index) == 48 {
while $index < len and str_get_utf8_byte_unsafe(digits, $index) == 48 {
$index = $index + 1
}

{
digits: Str.from_utf8_lossy(List.drop_first(bytes, $index)),
# the dropped prefix is ASCII zeros, so the cut is a UTF-8 boundary
digits: str_drop_first_bytes_unsafe(digits, $index),
digits_len: len - $index,
point: point - $index.to_i64_wrap(),
}
Expand All @@ -459,9 +457,9 @@ Builtin :: [].{
zero_count = point_u64 - digits_len
Ok(Str.concat(sign, Str.concat(digits, Str.repeat("0", zero_count))))
} else {
bytes = Str.to_utf8(digits)
before = Str.from_utf8_lossy(List.take_first(bytes, point_u64))
after = Str.from_utf8_lossy(List.drop_first(bytes, point_u64))
# digits holds only ASCII digits, so any cut is a UTF-8 boundary
before = str_substring_unsafe(digits, 0, point_u64)
after = str_drop_first_bytes_unsafe(digits, point_u64)

Ok(Str.concat(sign, Str.concat(before, Str.concat(".", after))))
}
Expand Down Expand Up @@ -1059,16 +1057,15 @@ Builtin :: [].{

is_json_number : Str -> Bool
is_json_number = |value| {
bytes = Str.to_utf8(value)
len = List.len(bytes)
len = Str.count_utf8_bytes(value)

if len == 0 {
return False
}

var $index = 0

first = list_get_unsafe(bytes, $index)
first = str_get_utf8_byte_unsafe(value, $index)

if first == 45 {
$index = $index + 1
Expand All @@ -1078,15 +1075,15 @@ Builtin :: [].{
}
}

int_first = list_get_unsafe(bytes, $index)
int_first = str_get_utf8_byte_unsafe(value, $index)

if int_first == 48 {
$index = $index + 1
} else if Json.is_json_digit_one_to_nine(int_first) {
$index = $index + 1

while $index < len {
byte = list_get_unsafe(bytes, $index)
byte = str_get_utf8_byte_unsafe(value, $index)

if Json.is_json_digit(byte) {
$index = $index + 1
Expand All @@ -1099,7 +1096,7 @@ Builtin :: [].{
}

if $index < len {
byte = list_get_unsafe(bytes, $index)
byte = str_get_utf8_byte_unsafe(value, $index)

if byte == 46 {
$index = $index + 1
Expand All @@ -1108,14 +1105,14 @@ Builtin :: [].{
return False
}

first_fraction_byte = list_get_unsafe(bytes, $index)
first_fraction_byte = str_get_utf8_byte_unsafe(value, $index)

if !Json.is_json_digit(first_fraction_byte) {
return False
}

while $index < len {
fraction_byte = list_get_unsafe(bytes, $index)
fraction_byte = str_get_utf8_byte_unsafe(value, $index)

if Json.is_json_digit(fraction_byte) {
$index = $index + 1
Expand All @@ -1127,7 +1124,7 @@ Builtin :: [].{
}

if $index < len {
byte = list_get_unsafe(bytes, $index)
byte = str_get_utf8_byte_unsafe(value, $index)

if Json.is_json_exponent_marker(byte) {
$index = $index + 1
Expand All @@ -1136,7 +1133,7 @@ Builtin :: [].{
return False
}

exponent_first_byte = list_get_unsafe(bytes, $index)
exponent_first_byte = str_get_utf8_byte_unsafe(value, $index)

if Json.is_json_sign(exponent_first_byte) {
$index = $index + 1
Expand All @@ -1146,14 +1143,14 @@ Builtin :: [].{
}
}

first_exponent_digit = list_get_unsafe(bytes, $index)
first_exponent_digit = str_get_utf8_byte_unsafe(value, $index)

if !Json.is_json_digit(first_exponent_digit) {
return False
}

while $index < len {
exponent_byte = list_get_unsafe(bytes, $index)
exponent_byte = str_get_utf8_byte_unsafe(value, $index)

if Json.is_json_digit(exponent_byte) {
$index = $index + 1
Expand Down
9 changes: 9 additions & 0 deletions test/alloc-count/app.roc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,15 @@ check_json_parse_allocs! = |input| {
skip_allocs = Host.alloc_count!() - skip_before
expect skip_allocs == 0
expect skipped == Ok({ a: 7 })

# a skipped numeric field: scalar validation must not allocate either
num_doc = Str.concat("{\"z\":", Str.concat(Str.repeat("7", 1 + Str.count_utf8_bytes(input) % 3), ",\"a\":1}"))
skip_num_before = Host.alloc_count!()
skipped_num : Try({ a : U64 }, Json.ParseErr)
skipped_num = Json.parse(num_doc)
skip_num_allocs = Host.alloc_count!() - skip_num_before
expect skip_num_allocs == 0
expect skipped_num == Ok({ a: 1 })
{}
}

Expand Down
Loading