diff --git a/ndc_stdlib/src/serde.rs b/ndc_stdlib/src/serde.rs index 312be94a..37a38021 100644 --- a/ndc_stdlib/src/serde.rs +++ b/ndc_stdlib/src/serde.rs @@ -1,81 +1,144 @@ -use anyhow::Context; +use anyhow::{Context, bail}; use ndc_core::hash_map::HashMap; use ndc_macros::export_module; use ndc_vm::value::{Object, Value}; use num::ToPrimitive; use serde_json::{Map, Number, Value as JsonValue, json}; +use std::collections::HashSet; use std::rc::Rc; +use std::str::FromStr; -fn value_to_json(value: Value) -> Result { +/// Converts a value to JSON. In strict mode (`lossy == false`) any value that +/// would not survive `json_decode(json_encode(value)) == value` is rejected +/// with an error. In lossy mode every value that can be reasonably represented +/// is accepted: rationals become floats, complex numbers become strings, +/// options are unwrapped, tuples and deques become arrays, heaps become arrays +/// in priority order, iterators are drained, non-string map keys are +/// stringified, and `None` and non-finite floats become null. +/// +/// `active` holds the containers currently being converted on the recursion +/// path, so a value that (transitively) contains itself is detected instead of +/// recursing forever. +fn value_to_json( + value: &Value, + lossy: bool, + active: &mut HashSet<*const Object>, +) -> Result { match value { - Value::None => Ok(JsonValue::Null), + Value::None if lossy => Ok(JsonValue::Null), + Value::None => bail!("cannot convert None to JSON, JSON null maps to the unit value ()"), Value::Bool(b) => Ok(json!(b)), Value::Int(i) => Ok(json!(i)), - Value::Float(f) => Ok(json!(f)), - Value::Object(obj) => match obj.as_ref() { - Object::Some(inner) => value_to_json(inner.clone()), - Object::BigInt(big_int) => { - use std::str::FromStr; - Number::from_str(&big_int.to_string()) - .map(JsonValue::Number) - .context("Cannot convert bigint to JSON number") + Value::Float(f) if f.is_finite() => Ok(json!(f)), + Value::Float(_) if lossy => Ok(JsonValue::Null), + Value::Float(f) => bail!("cannot convert non-finite float {f} to JSON"), + Value::Object(obj) => { + // Only these variants have interior mutability through which a + // value can contain itself; all other variants are leaves or + // immutable, so they never need to be tracked. + let cycle_guard = matches!( + obj.as_ref(), + Object::List(_) + | Object::Deque(_) + | Object::Map { .. } + | Object::MaxHeap(_) + | Object::MinHeap(_) + | Object::Iterator(_) + ); + if cycle_guard && !active.insert(Rc::as_ptr(obj)) { + bail!("cannot convert a value that contains itself to JSON"); } - Object::Rational(ratio) => Ok(json!(ratio.to_f64())), - Object::Complex(complex) => Ok(json!(format!("{complex}"))), - Object::String(s) => Ok(json!(&*s.borrow())), - Object::List(v) => Ok(JsonValue::Array( - v.borrow() - .iter() - .map(|v| value_to_json(v.clone())) - .collect::, _>>()?, - )), - Object::Tuple(v) => match v.len() { - 0 => Ok(JsonValue::Null), - _ => Ok(JsonValue::Array( - v.iter() - .map(|v| value_to_json(v.clone())) - .collect::, _>>()?, - )), - }, - Object::Map { entries, .. } => Ok(JsonValue::Object( - entries - .borrow() - .iter() - .map(|(key, value)| { - value_to_json(value.clone()).map(|value| (key.to_string(), value)) - }) - .collect::, _>>()?, - )), - Object::Iterator(i) => { - let mut out = Vec::new(); - let mut iter = i.borrow_mut(); - while let Some(v) = iter.next() { - out.push(value_to_json(v)?); - } - Ok(JsonValue::Array(out)) + let result = object_to_json(obj, lossy, active); + if cycle_guard { + active.remove(&Rc::as_ptr(obj)); } - Object::MaxHeap(h) => Ok(JsonValue::Array( - h.borrow() - .iter() - .map(|v| value_to_json(v.0.clone())) - .collect::, _>>()?, - )), - Object::MinHeap(h) => Ok(JsonValue::Array( - h.borrow() - .iter() - .map(|v| value_to_json(v.0.0.clone())) - .collect::, _>>()?, - )), - Object::Deque(d) => Ok(JsonValue::Array( - d.borrow() - .iter() - .map(|v| value_to_json(v.clone())) - .collect::, _>>()?, - )), - Object::Function(_) | Object::OverloadSet { .. } => { - Err(anyhow::anyhow!("Unable to serialize function")) + result + } + } +} + +fn object_to_json( + obj: &Rc, + lossy: bool, + active: &mut HashSet<*const Object>, +) -> Result { + let values_to_array = |values: &mut dyn Iterator, + active: &mut HashSet<*const Object>| { + values + .map(|v| value_to_json(v, lossy, active)) + .collect::, _>>() + .map(JsonValue::Array) + }; + + match obj.as_ref() { + Object::BigInt(big_int) => Number::from_str(&big_int.to_string()) + .map(JsonValue::Number) + .context("cannot convert bigint to JSON number"), + Object::Rational(ratio) if lossy => Ok(json!(ratio.to_f64())), + Object::Rational(_) => { + bail!("cannot convert a rational number to JSON, convert it to a float first") + } + Object::Complex(complex) if lossy => Ok(json!(format!("{complex}"))), + Object::Complex(_) => bail!("cannot convert a complex number to JSON"), + Object::Some(inner) if lossy => value_to_json(inner, lossy, active), + Object::Some(_) => bail!("cannot convert an option to JSON, unwrap it first"), + Object::Iterator(i) if lossy => { + let mut out = Vec::new(); + let mut iter = i.borrow_mut(); + while let Some(v) = iter.next() { + out.push(value_to_json(&v, lossy, active)?); } - }, + Ok(JsonValue::Array(out)) + } + Object::Iterator(_) => { + bail!("cannot convert an iterator to JSON, collect it into a list first") + } + Object::MaxHeap(h) if lossy => { + // Priority order: the order `pop` would produce + let mut sorted = h.borrow().clone().into_sorted_vec(); + sorted.reverse(); + values_to_array(&mut sorted.iter().map(|v| &v.0), active) + } + Object::MinHeap(h) if lossy => { + let mut sorted = h.borrow().clone().into_sorted_vec(); + sorted.reverse(); + values_to_array(&mut sorted.iter().map(|v| &v.0.0), active) + } + Object::MaxHeap(_) | Object::MinHeap(_) => { + bail!("cannot convert a heap to JSON, convert it to a list first") + } + Object::Function(_) | Object::OverloadSet { .. } => { + bail!("cannot convert a function to JSON") + } + Object::String(s) => Ok(json!(&*s.borrow())), + Object::Tuple(v) if v.is_empty() => Ok(JsonValue::Null), + Object::Tuple(v) if lossy => values_to_array(&mut v.iter(), active), + Object::Tuple(_) => bail!("cannot convert a tuple to JSON, convert it to a list first"), + Object::List(v) => values_to_array(&mut v.borrow().iter(), active), + Object::Deque(d) if lossy => values_to_array(&mut d.borrow().iter(), active), + Object::Deque(_) => bail!("cannot convert a deque to JSON, convert it to a list first"), + Object::Map { entries, default } => { + if default.is_some() && !lossy { + bail!("cannot convert a map with a default value to JSON"); + } + entries + .borrow() + .iter() + .map(|(key, value)| { + let key = match key { + _ if lossy => key.to_string(), + Value::Object(obj) => match obj.as_ref() { + Object::String(key) => key.borrow().clone(), + _ => bail!("cannot convert a map with non-string key {key} to JSON"), + }, + _ => bail!("cannot convert a map with non-string key {key} to JSON"), + }; + let value = value_to_json(value, lossy, active)?; + Ok((key, value)) + }) + .collect::, _>>() + .map(JsonValue::Object) + } } } @@ -84,12 +147,20 @@ fn json_to_value(value: JsonValue) -> Result { JsonValue::Null => Value::unit(), JsonValue::Bool(b) => Value::Bool(b), JsonValue::Number(n) => { - if let Some(i) = n.as_i64() { - Value::Int(i) - } else if let Some(f) = n.as_f64() { - Value::Float(f) + // With serde_json's arbitrary_precision feature the number's + // original text is preserved exactly, so integers of any size can + // be converted without going through a lossy f64. + let repr = n.to_string(); + if repr.contains(['.', 'e', 'E']) { + let float: f64 = repr.parse().context("cannot parse JSON number")?; + if !float.is_finite() { + bail!("JSON number {repr} does not fit in a float"); + } + Value::Float(float) + } else if let Ok(int) = repr.parse::() { + Value::Int(int) } else { - return Err(anyhow::anyhow!("Cannot parse JSON number")); + Value::bigint(repr.parse().context("cannot parse JSON number")?) } } JsonValue::String(s) => Value::string(s), @@ -109,15 +180,43 @@ fn json_to_value(value: JsonValue) -> Result { #[export_module] mod inner { - /// Converts a JSON string to a value + /// Converts a JSON string to a value: `json_decode("{\"a\": [1, null]}") == %{"a": [1, ()]}`. + /// + /// `null` becomes the unit value `()`, arrays become lists, objects become + /// maps with string keys, and integers too big for `Int` decode losslessly + /// to big integers. pub fn json_decode(input: &str) -> anyhow::Result { let json: JsonValue = serde_json::from_str(input)?; json_to_value(json) } - /// Converts the input value to JSON + /// Converts a value to a JSON string: `json_encode(%{"a": [1, ()]}) == "{\"a\":[1,null]}"`. + /// + /// Only values that decode back to an equal value are accepted, so this is + /// the exact inverse of `json_decode`. The unit value `()` converts to + /// `null`; sets convert to objects whose values are all `null`. Anything + /// else is rejected with an error: rationals, complex numbers, non-finite + /// floats, options (both `Some` and `None`), tuples, deques, iterators, + /// heaps, functions, maps with non-string keys or a default value, and + /// values that contain themselves. Use `json_encode_lossy` to convert + /// those anyway. pub fn json_encode(input: Value) -> anyhow::Result { - let v = value_to_json(input)?; + let v = value_to_json(&input, false, &mut HashSet::new())?; + Ok(v.to_string()) + } + + /// Converts any value to a JSON string, accepting values `json_encode` + /// rejects by degrading them: `json_encode_lossy((1, Some(1/2))) == "[1,0.5]"`. + /// + /// Rationals become floats, complex numbers become strings, `Some(x)` is + /// unwrapped to `x`, tuples and deques become arrays, heaps become arrays + /// in priority order, iterators are drained, non-string map keys are + /// stringified, and `None` and non-finite floats become `null`. There is + /// no lossy counterpart for `json_decode` because these conversions cannot + /// be reversed. Functions and values that contain themselves are still + /// errors. + pub fn json_encode_lossy(input: Value) -> anyhow::Result { + let v = value_to_json(&input, true, &mut HashSet::new())?; Ok(v.to_string()) } } diff --git a/tests/functional/programs/605_stdlib_serde/001_json.ndc b/tests/functional/programs/605_stdlib_serde/001_json.ndc index 904c0c92..87061edb 100644 --- a/tests/functional/programs/605_stdlib_serde/001_json.ndc +++ b/tests/functional/programs/605_stdlib_serde/001_json.ndc @@ -2,25 +2,23 @@ let object = %{ "foo": "bar", "int": 123, "float": 123.123, - "rational": 10/3, + "unit": (), + "set": %{"a", "b"}, "object": %{ "test": [1,2,3] }, - "set": %{1,2,3}, "list": [1,2,3,4], - "tuple": (1,2,3), }; let result = %{ "float": 123.123, "foo": "bar", - "rational": 3.3333333333333335, + "unit": (), + "set": %{"a", "b"}, "int": 123, - "tuple": [1,2,3], "object": %{ "test": [1,2,3] }, - "set": %{"1", "2", "3"}, // LMAO this is so lossy "list": [1,2,3,4] }; diff --git a/tests/functional/programs/605_stdlib_serde/003_numbers.ndc b/tests/functional/programs/605_stdlib_serde/003_numbers.ndc new file mode 100644 index 00000000..b00fb82b --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/003_numbers.ndc @@ -0,0 +1,13 @@ +// Big integers round-trip exactly instead of degrading to floats +let big = 2 ^ 100 + 1; +assert_eq(json_decode(json_encode(big)), big); +assert_eq(json_decode("123456789123456789123456789"), 123456789123456789123456789); + +// Numbers written with a decimal point or exponent decode to floats +assert_eq(json_decode("1e2"), 100.0); +assert_eq(json_decode("123.5"), 123.5); +assert_eq(json_decode(json_encode(0.1)), 0.1); + +// null and the unit value are the same thing on both sides +assert_eq(json_decode("null"), ()); +assert_eq(json_encode(()), "null"); diff --git a/tests/functional/programs/605_stdlib_serde/004_lossy.ndc b/tests/functional/programs/605_stdlib_serde/004_lossy.ndc new file mode 100644 index 00000000..7850aae1 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/004_lossy.ndc @@ -0,0 +1,41 @@ +// json_encode_lossy accepts everything json_encode rejects (except functions +// and cyclic values) by degrading it to the nearest JSON representation + +// Rationals become floats +assert_eq(json_encode_lossy(10/3), "3.3333333333333335"); + +// Complex numbers become strings +assert_eq(json_encode_lossy(5.0 + 3.1j), "\"5+3.1i\""); + +// Options are unwrapped +assert_eq(json_encode_lossy(Some(5)), "5"); +assert_eq(json_encode_lossy(None), "null"); + +// Sets become objects with null values +assert_eq(json_encode_lossy(%{"a", "b"}), "{\"a\":null,\"b\":null}"); + +// Non-finite floats and the unit value become null +assert_eq(json_encode_lossy(0.0 / 0.0), "null"); +assert_eq(json_encode_lossy((1, (), 2)), "[1,null,2]"); + +// Non-string map keys are stringified, map defaults are dropped +assert_eq(json_encode_lossy(%{1: "a"}), "{\"1\":\"a\"}"); +assert_eq(json_encode_lossy(%{: 0}), "{}"); + +// Iterators are drained +assert_eq(json_encode_lossy(1..4), "[1,2,3]"); + +// Heaps become arrays in priority order +let h = MaxHeap(); +h.push(1); h.push(3); h.push(2); +assert_eq(json_encode_lossy(h), "[3,2,1]"); + +let m = MinHeap(); +m.push(3); m.push(1); m.push(2); +assert_eq(json_encode_lossy(m), "[1,2,3]"); + +// Deques become arrays +let d = Deque(); +d.push_back(1); +d.push_front(0); +assert_eq(json_encode_lossy(d), "[0,1]"); diff --git a/tests/functional/programs/605_stdlib_serde/005_unit_and_sets.ndc b/tests/functional/programs/605_stdlib_serde/005_unit_and_sets.ndc new file mode 100644 index 00000000..3f304d7b --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/005_unit_and_sets.ndc @@ -0,0 +1,9 @@ +// The unit value () pairs with JSON null in both directions +assert_eq(json_encode(()), "null"); +assert_eq(json_decode("null"), ()); +assert_eq(json_encode([1, (), 2]), "[1,null,2]"); + +// A set is a map with unit values, so it round-trips as an object with nulls +assert_eq(json_encode(%{"a", "b"}), "{\"a\":null,\"b\":null}"); +assert_eq(json_decode(json_encode(%{"a", "b"})), %{"a", "b"}); +assert_eq(json_decode("{\"a\":null}"), %{"a": ()}); diff --git a/tests/functional/programs/605_stdlib_serde/010_encode_error_rational.ndc b/tests/functional/programs/605_stdlib_serde/010_encode_error_rational.ndc new file mode 100644 index 00000000..73020fdf --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/010_encode_error_rational.ndc @@ -0,0 +1,2 @@ +json_encode(10/3); +// expect-error: cannot convert a rational number to JSON diff --git a/tests/functional/programs/605_stdlib_serde/011_encode_error_complex.ndc b/tests/functional/programs/605_stdlib_serde/011_encode_error_complex.ndc new file mode 100644 index 00000000..2dc21979 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/011_encode_error_complex.ndc @@ -0,0 +1,2 @@ +json_encode(5.0 + 3.1j); +// expect-error: cannot convert a complex number to JSON diff --git a/tests/functional/programs/605_stdlib_serde/013_encode_error_heap.ndc b/tests/functional/programs/605_stdlib_serde/013_encode_error_heap.ndc new file mode 100644 index 00000000..e0c64f59 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/013_encode_error_heap.ndc @@ -0,0 +1,4 @@ +let h = MaxHeap(); +h.push(1); +json_encode(h); +// expect-error: cannot convert a heap to JSON diff --git a/tests/functional/programs/605_stdlib_serde/014_encode_error_iterator.ndc b/tests/functional/programs/605_stdlib_serde/014_encode_error_iterator.ndc new file mode 100644 index 00000000..2d1ac1f8 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/014_encode_error_iterator.ndc @@ -0,0 +1,2 @@ +json_encode(1..10); +// expect-error: cannot convert an iterator to JSON diff --git a/tests/functional/programs/605_stdlib_serde/015_encode_error_option.ndc b/tests/functional/programs/605_stdlib_serde/015_encode_error_option.ndc new file mode 100644 index 00000000..075e77b0 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/015_encode_error_option.ndc @@ -0,0 +1,2 @@ +json_encode(Some(5)); +// expect-error: cannot convert an option to JSON diff --git a/tests/functional/programs/605_stdlib_serde/016_encode_error_none.ndc b/tests/functional/programs/605_stdlib_serde/016_encode_error_none.ndc new file mode 100644 index 00000000..ad3ad108 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/016_encode_error_none.ndc @@ -0,0 +1,2 @@ +json_encode([1, None, 2]); +// expect-error: cannot convert None to JSON diff --git a/tests/functional/programs/605_stdlib_serde/017_encode_error_cycle.ndc b/tests/functional/programs/605_stdlib_serde/017_encode_error_cycle.ndc new file mode 100644 index 00000000..6aea15eb --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/017_encode_error_cycle.ndc @@ -0,0 +1,4 @@ +let l = [1, 2]; +l.push(l); +json_encode(l); +// expect-error: cannot convert a value that contains itself to JSON diff --git a/tests/functional/programs/605_stdlib_serde/018_encode_error_map_key.ndc b/tests/functional/programs/605_stdlib_serde/018_encode_error_map_key.ndc new file mode 100644 index 00000000..6f36b460 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/018_encode_error_map_key.ndc @@ -0,0 +1,2 @@ +json_encode(%{(1, 2): "a"}); +// expect-error: cannot convert a map with non-string key (1,2) to JSON diff --git a/tests/functional/programs/605_stdlib_serde/019_encode_error_nan.ndc b/tests/functional/programs/605_stdlib_serde/019_encode_error_nan.ndc new file mode 100644 index 00000000..ad91b870 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/019_encode_error_nan.ndc @@ -0,0 +1,2 @@ +json_encode(0.0 / 0.0); +// expect-error: cannot convert non-finite float diff --git a/tests/functional/programs/605_stdlib_serde/020_encode_error_default_map.ndc b/tests/functional/programs/605_stdlib_serde/020_encode_error_default_map.ndc new file mode 100644 index 00000000..7c0d79a1 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/020_encode_error_default_map.ndc @@ -0,0 +1,2 @@ +json_encode(%{: 0}); +// expect-error: cannot convert a map with a default value to JSON diff --git a/tests/functional/programs/605_stdlib_serde/021_decode_error_overflow.ndc b/tests/functional/programs/605_stdlib_serde/021_decode_error_overflow.ndc new file mode 100644 index 00000000..ac7272fb --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/021_decode_error_overflow.ndc @@ -0,0 +1,2 @@ +json_decode("1e999"); +// expect-error: does not fit in a float diff --git a/tests/functional/programs/605_stdlib_serde/023_encode_error_tuple.ndc b/tests/functional/programs/605_stdlib_serde/023_encode_error_tuple.ndc new file mode 100644 index 00000000..bc65122f --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/023_encode_error_tuple.ndc @@ -0,0 +1,2 @@ +json_encode((1, 2, 3)); +// expect-error: cannot convert a tuple to JSON diff --git a/tests/functional/programs/605_stdlib_serde/024_encode_error_deque.ndc b/tests/functional/programs/605_stdlib_serde/024_encode_error_deque.ndc new file mode 100644 index 00000000..dfb253a8 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/024_encode_error_deque.ndc @@ -0,0 +1,4 @@ +let d = Deque(); +d.push_back(1); +json_encode(d); +// expect-error: cannot convert a deque to JSON diff --git a/tests/functional/programs/605_stdlib_serde/025_lossy_error_cycle.ndc b/tests/functional/programs/605_stdlib_serde/025_lossy_error_cycle.ndc new file mode 100644 index 00000000..88478a7d --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/025_lossy_error_cycle.ndc @@ -0,0 +1,4 @@ +let l = [1, 2]; +l.push(l); +json_encode_lossy(l); +// expect-error: cannot convert a value that contains itself to JSON diff --git a/tests/functional/programs/605_stdlib_serde/026_lossy_error_heap_cycle.ndc b/tests/functional/programs/605_stdlib_serde/026_lossy_error_heap_cycle.ndc new file mode 100644 index 00000000..8dd784c2 --- /dev/null +++ b/tests/functional/programs/605_stdlib_serde/026_lossy_error_heap_cycle.ndc @@ -0,0 +1,4 @@ +let h = MaxHeap(); +h.push(h); +json_encode_lossy(h); +// expect-error: cannot convert a value that contains itself to JSON