From 6a51d18aa74518a221cc942754732640141a71cb Mon Sep 17 00:00:00 2001 From: Jiaan Geng Date: Thu, 7 Apr 2022 08:35:13 +0800 Subject: [PATCH 1/3] [SPARK-38391][SPARK-38768][SQL][FOLLOWUP] Add comments for pushLimit and pushTopN of PushDownUtils --- .../sql/execution/datasources/v2/PushDownUtils.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala index 37c180ef5d353..9f9a0ad2831f1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala @@ -116,7 +116,11 @@ object PushDownUtils extends PredicateHelper { } /** - * Pushes down LIMIT to the data source Scan + * Pushes down LIMIT to the data source Scan. + * + * @return the tuple of Boolean. The first Boolean value represents whether to push down, and + * the second Boolean value represents whether to push down partially which means to + * keep the `Limit`. */ def pushLimit(scanBuilder: ScanBuilder, limit: Int): (Boolean, Boolean) = { scanBuilder match { @@ -128,6 +132,10 @@ object PushDownUtils extends PredicateHelper { /** * Pushes down top N to the data source Scan + * + * @return the tuple of Boolean. The first Boolean value represents whether to push down, and + * the second Boolean value represents whether to push down partially which means to + * keep the `Sort` and `Limit`. */ def pushTopN( scanBuilder: ScanBuilder, From d368a1f7862e2b923679d08f2cffde1f9a60cada Mon Sep 17 00:00:00 2001 From: Jiaan Geng Date: Thu, 7 Apr 2022 08:40:04 +0800 Subject: [PATCH 2/3] Update code --- .../spark/sql/execution/datasources/v2/PushDownUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala index 9f9a0ad2831f1..98786aa8b030b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala @@ -131,7 +131,7 @@ object PushDownUtils extends PredicateHelper { } /** - * Pushes down top N to the data source Scan + * Pushes down top N to the data source Scan. * * @return the tuple of Boolean. The first Boolean value represents whether to push down, and * the second Boolean value represents whether to push down partially which means to From 385759b8e1ecb5a2d5cd0e54023c8c84c05e5541 Mon Sep 17 00:00:00 2001 From: Jiaan Geng Date: Sun, 10 Apr 2022 12:29:08 +0800 Subject: [PATCH 3/3] Update code --- .../sql/execution/datasources/v2/PushDownUtils.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala index 98786aa8b030b..aac7c3c1cf776 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala @@ -119,8 +119,8 @@ object PushDownUtils extends PredicateHelper { * Pushes down LIMIT to the data source Scan. * * @return the tuple of Boolean. The first Boolean value represents whether to push down, and - * the second Boolean value represents whether to push down partially which means to - * keep the `Limit`. + * the second Boolean value represents whether to push down partially, which means + * Spark will keep the Limit and do it again. */ def pushLimit(scanBuilder: ScanBuilder, limit: Int): (Boolean, Boolean) = { scanBuilder match { @@ -134,8 +134,8 @@ object PushDownUtils extends PredicateHelper { * Pushes down top N to the data source Scan. * * @return the tuple of Boolean. The first Boolean value represents whether to push down, and - * the second Boolean value represents whether to push down partially which means to - * keep the `Sort` and `Limit`. + * the second Boolean value represents whether to push down partially, which means + * Spark will keep the Sort and Limit and do it again. */ def pushTopN( scanBuilder: ScanBuilder,