Skip to content

Commit

Permalink
FIX-#6936: Fix 'read_parquet' when dataset is created with 'to_parque…
Browse files Browse the repository at this point in the history
…t' and 'index=False' (#6937)

Signed-off-by: Anatoly Myachev <anatoly.myachev@intel.com>
  • Loading branch information
anmyachev committed Feb 16, 2024
1 parent 56fb47e commit 3f9a733
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
1 change: 1 addition & 0 deletions modin/core/io/column_stores/parquet_dispatcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,7 @@ def build_query_compiler(cls, dataset, columns, index_columns, **kwargs):
if (
dataset.pandas_metadata
and "column_indexes" in dataset.pandas_metadata
and len(dataset.pandas_metadata["column_indexes"]) == 1
and dataset.pandas_metadata["column_indexes"][0]["numpy_type"] == "int64"
):
columns = pandas.Index(columns).astype("int64").to_list()
Expand Down
8 changes: 6 additions & 2 deletions modin/pandas/test/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -2029,15 +2029,19 @@ def test_read_parquet_5767(self, tmp_path, engine):
# both Modin and pandas read column "b" as a category
df_equals(test_df, read_df.astype("int64"))

def test_read_parquet_6855(self, tmp_path, engine):
@pytest.mark.parametrize("index", [False, True])
def test_read_parquet_6855(self, tmp_path, engine, index):
if engine == "fastparquet":
pytest.skip("integer columns aren't supported")
test_df = pandas.DataFrame(np.random.rand(10**2, 10))
path = tmp_path / "data"
path.mkdir()
file_name = "issue6855.parquet"
test_df.to_parquet(path / file_name, engine=engine)
test_df.to_parquet(path / file_name, index=index, engine=engine)
read_df = pd.read_parquet(path / file_name, engine=engine)
if not index:
# In that case pyarrow cannot preserve index dtype
read_df.columns = pandas.Index(read_df.columns).astype("int64").to_list()
df_equals(test_df, read_df)

def test_read_parquet_s3_with_column_partitioning(
Expand Down

0 comments on commit 3f9a733

Please sign in to comment.