Skip to content

Commit

Permalink
[SPARK-42929][CONNECT][FOLLOWUP] Rename isBarrier to barrier
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?

This is a follow-up of #40559 and #40571.

Renames `isBarrier` to `barrier` in Spark Connect, too.

### Why are the changes needed?

#40571 changed the argument name from `isBarrier` to `barrier`, so Spark Connect should follow it.

### Does this PR introduce _any_ user-facing change?

Yes, it renames the parameter.

### How was this patch tested?

Existing tests.

Closes #40579 from ueshin/issues/SPARK-42929/barrier.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
  • Loading branch information
ueshin authored and HyukjinKwon committed Mar 29, 2023
1 parent 6e4c352 commit aacac46
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 10 deletions.
Expand Up @@ -795,7 +795,7 @@ message MapPartitions {
// (Required) Input user-defined function.
CommonInlineUserDefinedFunction func = 2;

// (Optional) isBarrier.
// (Optional) Whether to use barrier mode execution or not.
optional bool is_barrier = 3;
}

Expand Down
14 changes: 6 additions & 8 deletions python/pyspark/sql/connect/dataframe.py
Expand Up @@ -1645,7 +1645,7 @@ def _map_partitions(
func: "PandasMapIterFunction",
schema: Union[StructType, str],
evalType: int,
is_barrier: bool,
barrier: bool,
) -> "DataFrame":
from pyspark.sql.connect.udf import UserDefinedFunction

Expand All @@ -1660,7 +1660,7 @@ def _map_partitions(

return DataFrame.withPlan(
plan.MapPartitions(
child=self._plan, function=udf_obj, cols=self.columns, is_barrier=is_barrier
child=self._plan, function=udf_obj, cols=self.columns, is_barrier=barrier
),
session=self._session,
)
Expand All @@ -1669,21 +1669,19 @@ def mapInPandas(
self,
func: "PandasMapIterFunction",
schema: Union[StructType, str],
is_barrier: bool = False,
barrier: bool = False,
) -> "DataFrame":
return self._map_partitions(
func, schema, PythonEvalType.SQL_MAP_PANDAS_ITER_UDF, is_barrier
)
return self._map_partitions(func, schema, PythonEvalType.SQL_MAP_PANDAS_ITER_UDF, barrier)

mapInPandas.__doc__ = PySparkDataFrame.mapInPandas.__doc__

def mapInArrow(
self,
func: "ArrowMapIterFunction",
schema: Union[StructType, str],
is_barrier: bool = False,
barrier: bool = False,
) -> "DataFrame":
return self._map_partitions(func, schema, PythonEvalType.SQL_MAP_ARROW_ITER_UDF, is_barrier)
return self._map_partitions(func, schema, PythonEvalType.SQL_MAP_ARROW_ITER_UDF, barrier)

mapInArrow.__doc__ = PySparkDataFrame.mapInArrow.__doc__

Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/sql/connect/proto/relations_pb2.pyi
Expand Up @@ -2767,7 +2767,7 @@ class MapPartitions(google.protobuf.message.Message):
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
is_barrier: builtins.bool
"""(Optional) isBarrier."""
"""(Optional) Whether to use barrier mode execution or not."""
def __init__(
self,
*,
Expand Down

0 comments on commit aacac46

Please sign in to comment.