From 48a22465a3d9a865cae0972b03183283fea0a77b Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 6 Jun 2026 17:15:04 -0700 Subject: [PATCH] [SPARK-57297] Add a test that SQL execution description respects spark.sql.redaction.string.regex --- .../sql/execution/SQLExecutionSuite.scala | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala index a85adc9ebf816..638d69fd7191d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.plans.logical.OneRowRelation import org.apache.spark.sql.classic import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.util.ThreadUtils import org.apache.spark.util.Utils.REDACTION_REPLACEMENT_TEXT @@ -396,6 +397,30 @@ class SQLExecutionSuite extends SparkFunSuite with SQLConfHelper { spark.stop() } } + + test("SQL execution description should respect spark.sql.redaction.string.regex") { + val spark = SparkSession.builder().master("local[*]").appName("test").getOrCreate() + try { + withSQLConf(SQLConf.SQL_STRING_REDACTION_PATTERN.key -> "password=([^\\s]+)") { + var sqlExecutionDescription: String = null + spark.sparkContext.addSparkListener(new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = event match { + case e: SparkListenerSQLExecutionStart => + sqlExecutionDescription = e.description + case _ => + } + }) + + val sqlStatement = "SELECT 'password=secret123'" + spark.sparkContext.setJobDescription(sqlStatement) + spark.sql(sqlStatement).collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + assert(sqlExecutionDescription === s"SELECT '$REDACTION_REPLACEMENT_TEXT") + } + } finally { + spark.stop() + } + } } object SQLExecutionSuite {