Skip to content

Commit

Permalink
FIX-modin-project#6879 : Convert the right DF to single partition bef…
Browse files Browse the repository at this point in the history
…ore broadcasting

Signed-off-by: arunjose696 <arunjose696@gmail.com>
  • Loading branch information
arunjose696 committed Jan 25, 2024
1 parent f6b31d6 commit 5331555
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 1 deletion.
31 changes: 31 additions & 0 deletions modin/core/dataframe/pandas/dataframe/dataframe.py
Expand Up @@ -2656,6 +2656,37 @@ def explode(self, axis: Union[int, Axis], func: Callable) -> "PandasDataframe":
partitions, new_index, new_columns, row_lengths, column_widths
)

def force_materialization(self) -> "PandasDataframe":
"""
Materialize axis partitions into a single partition.
Applies the identity function first across rows, and thereafter across columns
to get a df single partition.
Returns
-------
PandasDataframe
An PandasDataframe containing only a single partition.
"""
single_col_aggregated_df = self.apply_full_axis(
axis=0,
func=lambda x: x,
keep_partitioning=False,
num_splits=1,
sync_labels=True,
pass_axis_lengths_to_partitions=False,
)
single_partition_df = single_col_aggregated_df.apply_full_axis(
axis=1,
func=lambda x: x,
keep_partitioning=False,
num_splits=1,
sync_labels=True,
pass_axis_lengths_to_partitions=False,
)
return single_partition_df

@lazy_metadata_decorator(apply_axis="both")
def apply_full_axis(
self,
Expand Down
2 changes: 1 addition & 1 deletion modin/core/storage_formats/pandas/query_compiler.py
Expand Up @@ -520,7 +520,7 @@ def merge(self, right, **kwargs):
left_index = kwargs.get("left_index", False)
right_index = kwargs.get("right_index", False)
sort = kwargs.get("sort", False)

right = self.__constructor__(right._modin_frame.force_materialization())
if how in ["left", "inner"] and left_index is False and right_index is False:
kwargs["sort"] = False

Expand Down

0 comments on commit 5331555

Please sign in to comment.