From 43ca427368fb0ae62295b7c636946f2bde89324a Mon Sep 17 00:00:00 2001 From: Kevin Wilfong Date: Tue, 1 Sep 2026 10:37:49 -0700 Subject: [PATCH] cast null as complex --- .../gluten/execution/MiscOperatorSuite.scala | 30 +++++++++++ .../SubstraitToVeloxPlanValidator.cc | 8 +++ .../tests/Substrait2VeloxPlanValidatorTest.cc | 53 +++++++++++++++++++ .../utils/velox/VeloxTestSettings.scala | 2 + .../utils/velox/VeloxTestSettings.scala | 2 + .../utils/velox/VeloxTestSettings.scala | 1 + 6 files changed, 96 insertions(+) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index 6ddeaad2ef..948df8f8a3 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -22,6 +22,7 @@ import org.apache.gluten.expression.VeloxDummyExpression import org.apache.spark.SparkConf import org.apache.spark.shuffle.GlutenShuffleUtils import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQEShuffleReadExec, ColumnarAQEShuffleReadExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.joins.BaseJoinExec @@ -1957,6 +1958,35 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } } + test("cast null type to complex type") { + // An outer join whose right side turns out to be empty is replaced with a projection of a + // null cast to the type of each of that side's output attributes. Here the right side is only + // known to be empty once its stage has run, so AQE adds the casts after constant folding and + // they reach the backend as casts from the null type rather than as typed null literals. + val query = + """ + |select l.l_orderkey, r.arr, r.m, r.s + |from lineitem l left outer join ( + | select l_orderkey, array(l_partkey) as arr, map('k', l_partkey) as m, + | struct(l_partkey as a) as s + | from lineitem where l_orderkey < 0 + |) r on l.l_orderkey = r.l_orderkey + |""".stripMargin + runQueryAndCompare(query) { + df => + val plan = df.queryExecution.executedPlan + val castsToComplexTypes = collect(plan) { case p: ProjectExecTransformer => p } + .flatMap(_.projectList) + .flatMap(_.collect { case c: Cast if c.child.dataType == NullType => c.dataType }) + assert( + castsToComplexTypes.exists(_.isInstanceOf[ArrayType]), + s"Expect the null casts to be offloaded in:\n$plan") + // The casts must run natively rather than being split out to the JVM. + assert(collect(plan) { case p: ColumnarPartialProjectExec => p }.isEmpty) + assert(collect(plan) { case p: ProjectExec => p }.isEmpty) + } + } + test("timestamp broadcast join") { spark.range(0, 5).createOrReplaceTempView("right") spark.sql("SELECT id, timestamp_micros(id) as ts from right").createOrReplaceTempView("left") diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index 9896b43e54..2b36c1ff41 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -247,6 +247,14 @@ bool SubstraitToVeloxPlanValidator::isAllowedCast(const TypePtr& fromType, const return false; } + // Casting from UNKNOWN, e.g. a null constant, is allowed for any target type, + // including complex ones. The input is all nulls, so Velox short-circuits the + // cast to a null constant of the target type without ever looking at the + // input values. + if (fromType->kind() == TypeKind::UNKNOWN) { + return true; + } + // Limited support for DATE to X. if (fromType->isDate() && !toType->isTimestamp() && !toType->isVarchar()) { return false; diff --git a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc index a1147be798..c8c41e4e16 100644 --- a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc +++ b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc @@ -150,4 +150,57 @@ TEST_F(Substrait2VeloxPlanValidatorTest, aggregateMaskMustBeTopLevelField) { EXPECT_FALSE(validatePlan(nestedPlan)); } +TEST_F(Substrait2VeloxPlanValidatorTest, castFromUnknown) { + const auto validateCast = [&](const RowTypePtr& inputType, + const std::function& setInput, + const std::function& setToType) { + ::substrait::Expression expression; + auto* cast = expression.mutable_cast(); + setInput(cast->mutable_input()); + setToType(cast->mutable_type()); + + auto planValidator = std::make_shared(pool_.get()); + return planValidator->validate(expression, inputType, {}); + }; + + // A null constant, e.g. Spark's NullType, is expressed as the Nothing type. + const auto setNullInput = [](::substrait::Expression* input) { + input->mutable_literal()->mutable_null()->mutable_nothing(); + }; + const auto setNullableI32 = [](::substrait::Type* type) { + type->mutable_i32()->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE); + }; + + // Casting from Nothing is allowed for both scalar and complex target types. + EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, setNullableI32)); + + EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, [&](::substrait::Type* type) { + auto* list = type->mutable_list(); + list->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE); + setNullableI32(list->mutable_type()); + })); + + EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, [&](::substrait::Type* type) { + auto* map = type->mutable_map(); + map->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE); + setNullableI32(map->mutable_key()); + setNullableI32(map->mutable_value()); + })); + + EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, [&](::substrait::Type* type) { + auto* structType = type->mutable_struct_(); + structType->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE); + structType->add_names(""); + setNullableI32(structType->add_types()); + })); + + // Casting a complex type to an unrelated type is still not allowed. + EXPECT_FALSE(validateCast( + ROW({"a"}, {ARRAY(INTEGER())}), + [](::substrait::Expression* input) { + input->mutable_selection()->mutable_direct_reference()->mutable_struct_field()->set_field(0); + }, + setNullableI32)); +} + } // namespace gluten diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 538ca28417..8e8ea39605 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -942,6 +942,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("NOT NULL checks for atomic top-level fields (byPosition)") .exclude("NOT NULL checks for nested struct fields (byName)") .exclude("NOT NULL checks for nested struct fields (byPosition)") + .exclude("NOT NULL checks for nested structs, arrays, maps (byName)") + .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)") .exclude("NOT NULL checks for nullable array with required element (byPosition)") .exclude("not null checks for fields inside nullable array (byPosition)") enableSuite[GlutenTableOptionsConstantFoldingSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 053369d2be..9841eade00 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1225,6 +1225,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("NOT NULL checks for atomic top-level fields (byPosition)") .exclude("NOT NULL checks for nested struct fields (byName)") .exclude("NOT NULL checks for nested struct fields (byPosition)") + .exclude("NOT NULL checks for nested structs, arrays, maps (byName)") + .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)") .exclude("NOT NULL checks for nullable array with required element (byPosition)") .exclude("not null checks for fields inside nullable array (byPosition)") enableSuite[GlutenTableOptionsConstantFoldingSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 199068737c..419c71a073 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1223,6 +1223,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("NOT NULL checks for nested struct fields (byName)") .exclude("NOT NULL checks for nested struct fields (byPosition)") .exclude("NOT NULL checks for nested structs, arrays, maps (byName)") + .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)") .exclude("NOT NULL checks for nullable array with required element (byPosition)") .exclude("not null checks for fields inside nullable array (byPosition)") enableSuite[GlutenTableOptionsConstantFoldingSuite]