Skip to content

Commit

Permalink
fix v2
Browse files Browse the repository at this point in the history
  • Loading branch information
martindurant committed Aug 17, 2023
1 parent 43a2cee commit 7434ebc
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 2 deletions.
3 changes: 2 additions & 1 deletion fastparquet/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,8 @@ def read_data_page_v2(infile, schema_helper, se, data_header2, cmd,
# can read-into
into0 = ((use_cat or converts_inplace(se) and see)
and data_header2.num_nulls == 0
and max_rep == 0 and assign.dtype.kind != "O" and row_filter is None)
and max_rep == 0 and assign.dtype.kind != "O" and row_filter is None
and assign.dtype.kind not in "Mm") # TODO: this can be done in place but is complex
if row_filter is None:
row_filter = Ellipsis
# can decompress-into
Expand Down
3 changes: 2 additions & 1 deletion fastparquet/test/test_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,7 @@ def test_auto_null_object(tempdir):
df['bb'] = df['b'].astype('object')
df['aaa'] = df['a'].astype('object')
object_cols = ['d', 'ff', 'bb', 'aaa', 'aa']
test_cols = list(set(df) - set(object_cols)) + ['d']
test_cols = list(set(df) - set(object_cols) - {"c"}) + ['d']
fn = os.path.join(tmp, "test.parq")

with pytest.raises(ValueError):
Expand All @@ -573,6 +573,7 @@ def test_auto_null_object(tempdir):
assert col.repetition_type == parquet_thrift.FieldRepetitionType.OPTIONAL
df2 = pf.to_pandas(categories=['e'])

assert df2.c.equals(df.c)
tm.assert_frame_equal(df[test_cols], df2[test_cols], check_categorical=False,
check_dtype=False)
tm.assert_frame_equal(df[['bb']].astype('float64'), df2[['bb']])
Expand Down

0 comments on commit 7434ebc

Please sign in to comment.