Skip to content

Commit

Permalink
Merge pull request Point72#191 from Point72/ac/fix_to_json_parsing_fl…
Browse files Browse the repository at this point in the history 
…oats

Fix to_json serialization for floats
  • Loading branch information
@arhamchopra @Carreau
arhamchopra authored and Carreau committed May 13, 2024
2 parents c51a86b + 118a2fb commit cc60b87
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 5 deletions.
46 changes: 43 additions & 3 deletions cpp/csp/python/PyStructToJson.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,32 @@ inline rapidjson::Value toJson( const T& val, const CspType& typ, rapidjson::Doc
return rapidjson::Value( val );
}

// Helper function for parsing doubles
inline rapidjson::Value doubleToJson( const double& val, rapidjson::Document& doc )
{
// NOTE: Rapidjson adds support for this in a future release. Remove this when we upgrade rapidjson to a version
// after 07/16/2023 and use kWriteNanAndInfNullFlag in the writer.
//
// To be compatible with other JSON libraries, we cannot use the default approach that rapidjson has to
// serializing NaN, and (+/-)Infs. We need to manually convert them to NULLs. Rapidjson adds support for this
// in a future release.
if ( std::isnan( val ) || std::isinf( val ) )
{
return rapidjson::Value();
}
else
{
return rapidjson::Value( val );
}
}

// Helper function to convert doubles into json format recursively, by properly handlings NaNs, and Infs
template<>
inline rapidjson::Value toJson( const double& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable )
{
return doubleToJson( val, doc );
}

// Helper function to convert Enums into json format recursively
template<>
inline rapidjson::Value toJson( const CspEnum& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable )
Expand Down Expand Up @@ -183,7 +209,21 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc )
else if( PyFloat_Check( py_key ) )
{
auto key = PyFloat_AsDouble( py_key );
val.SetString( std::to_string( key ), doc.GetAllocator() );
auto json_obj = doubleToJson( key, doc );
if ( json_obj.IsNull() )
{
auto * str_obj = PyObject_Str( py_key );
Py_ssize_t len = 0;
const char * str = PyUnicode_AsUTF8AndSize( str_obj, &len );
CSP_THROW( ValueError, "Cannot serialize " + std::string( str ) + " to key in JSON" );
}
else
{
// Convert to string
std::stringstream s;
s << key;
val.SetString( s.str(), doc.GetAllocator() );
}
}
else
{
Expand Down Expand Up @@ -255,12 +295,12 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO
}
else if( PyFloat_Check( value ) )
{
return rapidjson::Value( fromPython<double>( value ) );
return doubleToJson( fromPython<double>( value ), doc );
}
else if( PyUnicode_Check( value ) )
{
Py_ssize_t len;
auto str = PyUnicode_AsUTF8AndSize( value , &len );
auto str = PyUnicode_AsUTF8AndSize( value, &len );
rapidjson::Value str_val;
str_val.SetString( str, len, doc.GetAllocator() );
return str_val;
Expand Down
52 changes: 50 additions & 2 deletions csp/tests/impl/test_struct.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1282,6 +1282,18 @@ class MyStruct(csp.Struct):
result_dict = {"b": False, "i": 456, "f": 1.73, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

test_struct = MyStruct(b=False, i=456, f=float("nan"), s="789")
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

test_struct = MyStruct(b=False, i=456, f=float("inf"), s="789")
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

test_struct = MyStruct(b=False, i=456, f=float("-inf"), s="789")
result_dict = {"b": False, "i": 456, "f": None, "s": "789"}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

def test_to_json_enums(self):
from enum import Enum as PyEnum

Expand Down Expand Up @@ -1434,8 +1446,13 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "l_any": l_l_i}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [12, False]))]]]
l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [12, False]]]]]]
l_any = [[1, float("nan")], [float("INFINITY"), float("-inf")]]
test_struct = MyStruct(i=456, l_any=l_any)
result_dict = {"i": 456, "l_any": [[1, None], [None, None]]}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [float("nan"), False]))]]]
l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [None, False]]]]]]
test_struct = MyStruct(i=456, l_any=l_any)
result_dict = {"i": 456, "l_any": l_any_result}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)
Expand All @@ -1444,6 +1461,7 @@ def test_to_json_dict(self):
class MyStruct(csp.Struct):
i: int = 123
d_i: typing.Dict[int, int]
d_f: typing.Dict[float, int]
d_dt: typing.Dict[str, datetime]
d_d_s: typing.Dict[str, typing.Dict[str, str]]
d_any: dict
Expand All @@ -1458,6 +1476,12 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "d_i": d_i_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_f=d_f)
result_dict = {"i": 456, "d_f": d_f_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

dt = datetime.now(tz=pytz.utc)
d_dt = {"d1": dt, "d2": dt}
test_struct = MyStruct(i=456, d_dt=d_dt)
Expand All @@ -1475,6 +1499,12 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "d_any": d_i_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
result_dict = {"i": 456, "d_any": d_f_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

dt = datetime.now(tz=pytz.utc)
d_dt = {"d1": dt, "d2": dt}
test_struct = MyStruct(i=456, d_any=d_dt)
Expand All @@ -1487,6 +1517,24 @@ class MyStruct(csp.Struct):
result_dict = {"i": 456, "d_any": d_any_res}
self.assertEqual(json.loads(test_struct.to_json()), result_dict)

d_f = {float("nan"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
with self.assertRaises(ValueError):
test_struct.to_json()

d_f = {float("inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
with self.assertRaises(ValueError):
test_struct.to_json()

d_f = {float("-inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7}
d_f_res = {str(k): v for k, v in d_f.items()}
test_struct = MyStruct(i=456, d_any=d_f)
with self.assertRaises(ValueError):
test_struct.to_json()

def test_to_json_struct(self):
class MySubSubStruct(csp.Struct):
b: bool = True
Expand Down

0 comments on commit cc60b87

Please sign in to comment.