PanicError in read_ndjson and polars.str.json_decode with empty struct #13433
Comments
|
The issue exists even with polars v 0.20.3 |
|
Hi @deep8324, the data schema of the DataFrame in your example is not well-defined, so it's no surprise that you get an error. It seems like the actual schema that the example seems to represent is the following: OrderedDict([('offer', Struct({'my_value': Int64, 'condition': List(Struct({'applicationId': Int64, 'conditionRequestReason': List(Struct({'a': Int64}))}))}))])The data for the list of struct with the field # pylint: disable=missing-class-docstring, missing-function-docstring
""" Test for reading NDJSON files with nested fields. """
import io
import json
import unittest
from typing import Sequence, Any, cast
import polars as pl
import polars.testing
class TestNdJson(unittest.TestCase):
def test_json_format(self):
well_defined_data = io.StringIO(
"""{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": []}]}}
{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{"a":0}]}]}}""")
# Initialize the frame from a sequence of dictionaries.
dicts = cast(Sequence[dict[str, Any]],
(json.loads(line) for line in well_defined_data))
frame_from_dicts = pl.from_dicts(dicts)
print(frame_from_dicts.schema)
# Initialize the frame from the NDJSON file.
frame_from_ndjson = pl.read_ndjson(well_defined_data)
print(frame_from_ndjson.schema)
# Compare the content of the frames.
polars.testing.assert_frame_equal(frame_from_dicts, frame_from_ndjson)
# Initialize the frame from a malformed NDJSON
incomplete_schema_data = io.StringIO(
"""{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{}]}]}}""")
frame_with_schema = pl.read_ndjson(incomplete_schema_data, schema=frame_from_ndjson.schema)
frame_with_schema_export = frame_with_schema.write_ndjson()
self.assertEqual(
'{"offer":{"my_value":0,"condition":[{"applicationId":0,"conditionRequestReason":[{"a":null}]}]}}\n',
frame_with_schema_export)
if __name__ == '__main__':
unittest.main()However, I can observe that without schema information, the behavior of def test_read_ndjson_list_of_awkward_struct(self):
input_data = io.StringIO(
"""{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{}]}]}}
{"offer": {"my_value": 0, "condition": [{"applicationId": 0, "conditionRequestReason": [{"a":0}]}]}}""")
# Initialize the frame from a sequence of dictionaries.
dicts = cast(Sequence[dict[str, Any]],
(json.loads(line) for line in input_data))
frame_from_dicts = pl.from_dicts(dicts)
print(frame_from_dicts.schema)
# Initialize the frame from the NDJSON file.
frame_from_ndjson = pl.read_ndjson(input_data)
print(frame_from_ndjson.schema)
# Compare the content of the frames.
polars.testing.assert_frame_equal(frame_from_dicts, frame_from_ndjson)The behavior of >>> frame_from_dicts
shape: (2, 1)
┌─────────────────────────┐
│ offer │
│ --- │
│ struct[2] │
╞═════════════════════════╡
│ {0,[{0,[{null,null}]}]} │
│ {0,[{0,[{null,0}]}]} │
└─────────────────────────┘
>>> frame_from_dicts.schema
OrderedDict([('offer', Struct({'my_value': Int64, 'condition': List(Struct({'applicationId': Int64, 'conditionRequestReason': List(Struct({'': Null, 'a': Int64}))}))}))]) |
|
Enclosed is a small simple example, where JSON decode doesn't determine the correct schema for a list of integer. It works for list of string, but not for list of integer. # pylint: disable=missing-class-docstring, missing-function-docstring, too-few-public-methods
""" Unit tests for the polars JSON decode functionality. """
import io
import pytest
import polars as pl
import polars.testing
class TestPolarsJsonDecode:
@pytest.mark.parametrize('test_name, input_ndjson', [
("str", """{"list_field": ["a", "b"]}
{"list_field": []}
{"list_field": ["c", "d", "e"]}"""),
("int", """{"list_field": [1, 2]}
{"list_field": []}
{"list_field": [4, 5, 6]}""")
])
def test_json_decode_list_of_basic_type(self, test_name, input_ndjson):
print(f"Reading list of {test_name}...")
input_buf = io.StringIO(input_ndjson)
frame_from_ndjson = pl.read_ndjson(input_buf)
# pylint: disable-next=assignment-from-no-return
series_with_json = pl.Series(values=input_buf).str.strip_chars_start()
series_decoded_unnested = series_with_json.str.json_decode().struct.unnest()
assert frame_from_ndjson.schema == series_decoded_unnested.schema
polars.testing.assert_frame_equal(frame_from_ndjson, series_decoded_unnested)
if __name__ == '__main__':
pytest.main() |
|
I just did some tests with polars 0.20.6 (and pyarrow 13.0.0)
Is polars using arrow for decoding JSON input? |
|
The following minimal test reproduces the PanicException that @deep8324 reported above. import io
import json
import pytest
import polars as pl
@pytest.mark.parametrize('input_ndjson', [
'{"bar": [{}]}', # nested_null
'{"foo":[{"bar":[{}]}]}' # nested_nested_null
])
def test_ndjson_nested_nested_null(input_ndjson):
""" Test whether read_ndjson and from_dicts imports nested null structs in the same way. """
buffer = io.StringIO(input_ndjson)
json_obj = json.loads(next(buffer))
df_from_dicts = pl.from_dicts([json_obj])
df_from_ndjson = pl.read_ndjson(buffer)
assert df_from_dicts.schema == df_from_ndjson.schemaFirst we have the Expected :OrderedDict([('bar', List(Struct({})))])
Actual :OrderedDict([('bar', List(Struct({'': Null})))])And then we have the The unexpected difference is in the line that I highlighted with the arrows. Instead of an empty struct, the code receives a struct with a field that has the name "" and a null value. polars unit tested behavior of the import of empty structs from dictionaries and the import of empty structs from NDJSON is inconsistent. Importing Feel free to add this test to the polars unit tests. |
|
It still reproduces in polars 0.20.10. I think it really comes down to the fact that polars is treating the empty object "{}" in a different way, depending on where it shows up in the schema. def test_ndjson_empty_object() -> None:
"""
The actual type of `empty_object_column` (tested in `test_ndjson_null_buffer()`) is inconsistent with
the actual type of `nested_empty_object_column` (tested in `test_ndjson_nested_null()`).
"""
data = io.BytesIO(
b"""\
{"id": 1, "empty_object_column": {}, "nested_empty_object_column": [{}]}
{"id": 2, "empty_object_column": {}, "nested_empty_object_column": [{}]}
{"id": 3, "empty_object_column": {}, "nested_empty_object_column": [{}]}
"""
)
assert pl.read_ndjson(data).schema == {
"id": pl.Int64,
"empty_object_column": pl.Struct([]), # or pl.Struct([pl.Field("", pl.Null)]) ?
"nested_empty_object_column": pl.List(pl.Struct([]))
}The test above fails with the output: @ritchie46 : It seems like you wrote the I'm mentioning the two of you, since I would be interested to know which of the two behaviors is the expected one. |
Checks
I have checked that this issue has not already been reported.
I have confirmed this bug exists on the latest version of Polars.
Reproducible example
Log output
Issue description
The issues happens when a valid json file with an empty struct as follows
{"offer":{"my_value":0,"condition":[{"applicationId":0,"conditionRequestReason":[{}]}]}}
is read using polars.read_ndjson it ends up in panic error.
Same issue appears if such a string is present as string and polars.str.json_decode() is used
Expected behavior
Expected behavior
the json file with value {"offer":{"my_value":0,"condition":[{"applicationId":0,"conditionRequestReason":[{}]}]}},
is read by read_ndjson and a polars data frame is produced
┌────────────────────┐
│ offer │
│ --- │
│ struct[2] │
╞════════════════════╡
│ {0,[{0,[{null}]}]} │
└────────────────────┘
Installed versions
The text was updated successfully, but these errors were encountered: