redact nondeterministic output

commit commit commit commit commit commit commit commit commit
apache · Mar 21, 2023 · 5d987f4 · 5d987f4
1 parent d9d8abd
commit 5d987f4
Show file tree

Hide file tree

Showing 47 changed files with 548 additions and 1,073 deletions.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -238,6 +238,16 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]]
     transformAllExpressionsWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule)
   }
 
+  /**
+   * A variant of [[transformAllExpressions]] which considers plan nodes inside subqueries as well.
+   */
+  def transformAllExpressionsWithSubqueries(
+    rule: PartialFunction[Expression, Expression]): this.type = {
+    transformWithSubqueries {
+      case q => q.transformExpressions(rule).asInstanceOf[PlanType]
+    }.asInstanceOf[this.type]
+  }
+
   /**
    * Returns the result of running [[transformExpressionsWithPruning]] on this node
    * and all its children. Note that this method skips expressions inside subqueries.

diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/array.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/array.sql.out
@@ -129,13 +129,7 @@ select
   array_contains(timestamp_array, timestamp '2016-11-15 20:54:00.000'), array_contains(timestamp_array, timestamp '2016-01-01 20:54:00.000')
 from primitive_arrays
 -- !query analysis
-Project [array_contains(boolean_array#x, true) AS array_contains(boolean_array, true)#x, array_contains(boolean_array#x, false) AS array_contains(boolean_array, false)#x, array_contains(tinyint_array#x, 2) AS array_contains(tinyint_array, 2)#x, array_contains(tinyint_array#x, 0) AS array_contains(tinyint_array, 0)#x, array_contains(smallint_array#x, 2) AS array_contains(smallint_array, 2)#x, array_contains(smallint_array#x, 0) AS array_contains(smallint_array, 0)#x, array_contains(int_array#x, 2) AS array_contains(int_array, 2)#x, array_contains(int_array#x, 0) AS array_contains(int_array, 0)#x, array_contains(bigint_array#x, 2) AS array_contains(bigint_array, 2)#x, array_contains(bigint_array#x, 0) AS array_contains(bigint_array, 0)#x, array_contains(decimal_array#x, 9223372036854775809) AS array_contains(decimal_array, 9223372036854775809)#x, array_contains(decimal_array#x, cast(1 as decimal(19,0))) AS array_contains(decimal_array, 1)#x, array_contains(double_array#x, 2.0) AS array_contains(double_array, 2.0)#x, array_contains(double_array#x, 0.0) AS array_contains(double_array, 0.0)#x, array_contains(float_array#x, cast(2.0 as float)) AS array_contains(float_array, 2.0)#x, array_contains(float_array#x, cast(0.0 as float)) AS array_contains(float_array, 0.0)#x, array_contains(date_array#x, 2016-03-14) AS array_contains(date_array, DATE '2016-03-14')#x, array_contains(date_array#x, 2016-01-01) AS array_contains(date_array, DATE '2016-01-01')#x, array_contains(timestamp_array#x, 2016-11-15 20:54:00) AS array_contains(timestamp_array, TIMESTAMP '2016-11-15 20:54:00')#x, array_contains(timestamp_array#x, 2016-01-01 20:54:00) AS array_contains(timestamp_array, TIMESTAMP '2016-01-01 20:54:00')#x]
-+- SubqueryAlias primitive_arrays
-   +- View (`primitive_arrays`, [boolean_array#x,tinyint_array#x,smallint_array#x,int_array#x,bigint_array#x,decimal_array#x,double_array#x,float_array#x,date_array#x,timestamp_array#x])
-      +- Project [cast(boolean_array#x as array<boolean>) AS boolean_array#x, cast(tinyint_array#x as array<tinyint>) AS tinyint_array#x, cast(smallint_array#x as array<smallint>) AS smallint_array#x, cast(int_array#x as array<int>) AS int_array#x, cast(bigint_array#x as array<bigint>) AS bigint_array#x, cast(decimal_array#x as array<decimal(19,0)>) AS decimal_array#x, cast(double_array#x as array<double>) AS double_array#x, cast(float_array#x as array<float>) AS float_array#x, cast(date_array#x as array<date>) AS date_array#x, cast(timestamp_array#x as array<timestamp>) AS timestamp_array#x]
-         +- Project [boolean_array#x, tinyint_array#x, smallint_array#x, int_array#x, bigint_array#x, decimal_array#x, double_array#x, float_array#x, date_array#x, timestamp_array#x]
-            +- SubqueryAlias primitive_arrays
-               +- LocalRelation [boolean_array#x, tinyint_array#x, smallint_array#x, int_array#x, bigint_array#x, decimal_array#x, double_array#x, float_array#x, date_array#x, timestamp_array#x]
+[Analyzer test output redacted due to nondeterminism]
 
 
 -- !query
@@ -657,3 +651,66 @@ select array_append(array(CAST(NULL AS String)), CAST(NULL AS String))
 -- !query analysis
 Project [array_append(array(cast(null as string)), cast(null as string)) AS array_append(array(CAST(NULL AS STRING)), CAST(NULL AS STRING))#x]
 +- OneRowRelation
+
+
+-- !query
+select array_prepend(array(1, 2, 3), 4)
+-- !query analysis
+Project [array_prepend(array(1, 2, 3), 4) AS array_prepend(array(1, 2, 3), 4)#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(array('a', 'b', 'c'), 'd')
+-- !query analysis
+Project [array_prepend(array(a, b, c), d) AS array_prepend(array(a, b, c), d)#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(array(1, 2, 3, NULL), NULL)
+-- !query analysis
+Project [array_prepend(array(1, 2, 3, cast(null as int)), cast(null as int)) AS array_prepend(array(1, 2, 3, NULL), NULL)#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(array('a', 'b', 'c', NULL), NULL)
+-- !query analysis
+Project [array_prepend(array(a, b, c, cast(null as string)), cast(null as string)) AS array_prepend(array(a, b, c, NULL), NULL)#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(CAST(null AS ARRAY<String>), 'a')
+-- !query analysis
+Project [array_prepend(cast(null as array<string>), a) AS array_prepend(NULL, a)#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(CAST(null AS ARRAY<String>), CAST(null as String))
+-- !query analysis
+Project [array_prepend(cast(null as array<string>), cast(null as string)) AS array_prepend(NULL, CAST(NULL AS STRING))#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(array(), 1)
+-- !query analysis
+Project [array_prepend(cast(array() as array<int>), 1) AS array_prepend(array(), 1)#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(CAST(array() AS ARRAY<String>), CAST(NULL AS String))
+-- !query analysis
+Project [array_prepend(cast(array() as array<string>), cast(null as string)) AS array_prepend(array(), CAST(NULL AS STRING))#x]
++- OneRowRelation
+
+
+-- !query
+select array_prepend(array(CAST(NULL AS String)), CAST(NULL AS String))
+-- !query analysis
+Project [array_prepend(array(cast(null as string)), cast(null as string)) AS array_prepend(array(CAST(NULL AS STRING)), CAST(NULL AS STRING))#x]
++- OneRowRelation
diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/cast.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/cast.sql.out
@@ -397,13 +397,13 @@ org.apache.spark.sql.AnalysisException
 -- !query
 DESC FUNCTION boolean
 -- !query analysis
-DescribeFunctionCommand org.apache.spark.sql.catalyst.expressions.ExpressionInfo@64d4353, false
+DescribeFunctionCommand org.apache.spark.sql.catalyst.expressions.ExpressionInfo@xxxxxxxx, false
 
 
 -- !query
 DESC FUNCTION EXTENDED boolean
 -- !query analysis
-DescribeFunctionCommand org.apache.spark.sql.catalyst.expressions.ExpressionInfo@64d4353, true
+DescribeFunctionCommand org.apache.spark.sql.catalyst.expressions.ExpressionInfo@xxxxxxxx, true
 
 
 -- !query