Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-37013][SQL][FOLLOWUP] Add legacy flag for the breaking change of forbidding %0$ usage in format_string #36101

Closed
wants to merge 1 commit into from
Closed

[SPARK-37013][SQL][FOLLOWUP] Add legacy flag for the breaking change of forbidding %0$ usage in format_string #36101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion ...catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
View file
Open in desktop
Expand Up @@ -1808,7 +1808,9 @@ case class ParseUrl(children: Seq[Expression], failOnError: Boolean = SQLConf.ge
case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes {

require(children.nonEmpty, s"$prettyName() should take at least 1 argument")
checkArgumentIndexNotZero(children(0))
if (!SQLConf.get.getConf(SQLConf.ALLOW_ZERO_INDEX_IN_FORMAT_STRING)) {
checkArgumentIndexNotZero(children(0))
}


override def foldable: Boolean = children.forall(_.foldable)
Expand Down
11 changes: 11 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
View file
Open in desktop
Expand Up @@ -2015,6 +2015,17 @@ object SQLConf {
.booleanConf
.createWithDefault(false)

val ALLOW_ZERO_INDEX_IN_FORMAT_STRING =
buildConf("spark.sql.legacy.allowZeroIndexInFormatString")
.internal()
.doc("When false, the `strfmt` in `format_string(strfmt, obj, ...)` and " +
"`printf(strfmt, obj, ...)` will no longer support to use \"0$\" to specify the first " +
"argument, the first argument should always reference by \"1$\" when use argument index " +
"to indicating the position of the argument in the argument list.")
.version("3.3")
.booleanConf
.createWithDefault(false)

val USE_CURRENT_SQL_CONFIGS_FOR_VIEW =
buildConf("spark.sql.legacy.useCurrentConfigsForView")
.internal()
Expand Down
13 changes: 8 additions & 5 deletions sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
View file
Open in desktop
Expand Up @@ -19,6 +19,7 @@ package org.apache.spark.sql.errors

import org.apache.spark.sql.{AnalysisException, IntegratedUDFTestUtils, QueryTest}
import org.apache.spark.sql.functions.{grouping, grouping_id, sum}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession

case class StringLongClass(a: String, b: Long)
Expand Down Expand Up @@ -94,12 +95,14 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession {
}

test("ILLEGAL_SUBSTRING: the argument_index of string format is invalid") {
val e = intercept[AnalysisException] {
sql("select format_string('%0$s', 'Hello')")
withSQLConf(SQLConf.ALLOW_ZERO_INDEX_IN_FORMAT_STRING.key -> "false") {
val e = intercept[AnalysisException] {
sql("select format_string('%0$s', 'Hello')")
}
assert(e.errorClass === Some("ILLEGAL_SUBSTRING"))
assert(e.message ===
"The argument_index of string format cannot contain position 0$.")
}
assert(e.errorClass === Some("ILLEGAL_SUBSTRING"))
assert(e.message ===
"The argument_index of string format cannot contain position 0$.")
}

test("CANNOT_USE_MIXTURE: Using aggregate function with grouped aggregate pandas UDF") {
Expand Down