Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[VL] Enable array_size Spark function #5539

Closed
wants to merge 6 commits into from
Closed

[VL] Enable array_size Spark function #5539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7f83f0d
Add array_size
acvictor Apr 25, 2024
d6886c1
Remove array_size from 3.2
acvictor Apr 25, 2024
75171be
Merge branch 'main' of https://github.com/acvictor/gluten into acvict…
acvictor Apr 29, 2024
294f748
Fix format
acvictor Apr 29, 2024
b72757e
Update test
acvictor Apr 29, 2024
98afa37
Fix format
acvictor Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ object CHExpressionUtil {
ARRAY_EXCEPT -> DefaultValidator(),
ARRAY_REPEAT -> DefaultValidator(),
ARRAY_REMOVE -> DefaultValidator(),
ARRAY_SIZE -> DefaultValidator(),
DATE_FROM_UNIX_DATE -> DefaultValidator(),
UNIX_DATE -> DefaultValidator(),
MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
Expand Down
19 changes: 19 additions & 0 deletions backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package org.apache.gluten.execution

import org.apache.gluten.sql.shims.SparkShimLoader

import org.apache.spark.sql.types._

import java.sql.Timestamp
Expand Down Expand Up @@ -826,4 +828,21 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest {
}
}

test("test array_size") {
if (!SparkShimLoader.getSparkVersion.startsWith("3.2")) {
withTempPath {
path =>
Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null)
.toDF("value")
.write
.parquet(path.getCanonicalPath)

spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl")

runQueryAndCompare("select array_size(value) as res from array_tbl;") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}
}
}
}
1 change: 1 addition & 0 deletions shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ object ExpressionNames {
final val ARRAY_EXCEPT = "array_except"
final val ARRAY_REPEAT = "array_repeat"
final val ARRAY_REMOVE = "array_remove"
final val ARRAY_SIZE = "array_size"
final val FILTER = "filter"
final val FORALL = "forall"
final val EXISTS = "exists"
Expand Down
5 changes: 3 additions & 2 deletions shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package org.apache.gluten.sql.shims.spark33

import org.apache.gluten.execution.datasource.GlutenParquetWriterInjects
import org.apache.gluten.expression.{ExpressionNames, Sig}
import org.apache.gluten.expression.ExpressionNames.{CEIL, FLOOR, KNOWN_NULLABLE, TIMESTAMP_ADD}
import org.apache.gluten.expression.ExpressionNames.{ARRAY_SIZE, CEIL, FLOOR, KNOWN_NULLABLE, TIMESTAMP_ADD}
import org.apache.gluten.sql.shims.{ShimDescriptor, SparkShims}

import org.apache.spark._
Expand Down Expand Up @@ -69,7 +69,8 @@ class Spark33Shims extends SparkShims {
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[TimestampAdd](TIMESTAMP_ADD),
Sig[RoundFloor](FLOOR),
Sig[RoundCeil](CEIL)
Sig[RoundCeil](CEIL),
Sig[ArraySize](ARRAY_SIZE)
)
}

Expand Down
5 changes: 3 additions & 2 deletions shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package org.apache.gluten.sql.shims.spark34

import org.apache.gluten.expression.{ExpressionNames, Sig}
import org.apache.gluten.expression.ExpressionNames.KNOWN_NULLABLE
import org.apache.gluten.expression.ExpressionNames.{ARRAY_SIZE, KNOWN_NULLABLE}
import org.apache.gluten.sql.shims.{ShimDescriptor, SparkShims}

import org.apache.spark._
Expand Down Expand Up @@ -70,7 +70,8 @@ class Spark34Shims extends SparkShims {
Sig[Sec](ExpressionNames.SEC),
Sig[Csc](ExpressionNames.CSC),
Sig[KnownNullable](KNOWN_NULLABLE),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL)
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[ArraySize](ARRAY_SIZE)
)
}

Expand Down
5 changes: 4 additions & 1 deletion shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.gluten.sql.shims.spark35

import org.apache.gluten.expression.{ExpressionNames, Sig}
import org.apache.gluten.expression.ExpressionNames.ARRAY_SIZE
import org.apache.gluten.sql.shims.{ShimDescriptor, SparkShims}

import org.apache.spark._
Expand Down Expand Up @@ -70,7 +71,9 @@ class Spark35Shims extends SparkShims {
Sig[SplitPart](ExpressionNames.SPLIT_PART),
Sig[Sec](ExpressionNames.SEC),
Sig[Csc](ExpressionNames.CSC),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL))
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[ArraySize](ARRAY_SIZE)
)
}

override def aggregateExpressionMappings: Seq[Sig] = {
Expand Down