Skip to content

Commit

Permalink
* Speed up selecting row-groups (#856)
Browse files Browse the repository at this point in the history
* xfail warning test
  • Loading branch information
martindurant committed Mar 20, 2023
1 parent e4b16f8 commit ca4a4fd
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 5 deletions.
13 changes: 8 additions & 5 deletions fastparquet/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,11 +311,14 @@ def __getitem__(self, item):
new_rgs = self.row_groups[item]
if not isinstance(new_rgs, list):
new_rgs = [new_rgs]
new_pf = copy.deepcopy(self)
new_pf.fmd.row_groups = new_rgs
new_pf._set_attrs()
# would otherwise be "simple" when selecting one rg
new_pf.file_scheme = self.file_scheme
new_pf = object.__new__(ParquetFile)
fmd = copy.copy(self.fmd)
fmd.row_groups = new_rgs
new_pf.__setstate__(
{"fn": self.fn, "open": self.open, "fmd": fmd,
"pandas_nulls": self.pandas_nulls, "_base_dtype": self._base_dtype,
"tz": self.tz}
)
return new_pf

def __len__(self):
Expand Down
2 changes: 2 additions & 0 deletions fastparquet/test/test_dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from unittest import mock

import pandas as pd
import pytest
from numpy import empty as np_empty
from pandas.testing import assert_frame_equal

Expand Down Expand Up @@ -33,6 +34,7 @@ def test_empty():
assert len(views) == 5


@pytest.mark.xfail(reason="df._data is going away")
def test_empty_tz_utc():
with warnings.catch_warnings():
warnings.simplefilter("error")
Expand Down

0 comments on commit ca4a4fd

Please sign in to comment.