From 564799021b855f8c3a7968f236ee792fcde29deb Mon Sep 17 00:00:00 2001 From: turbofei Date: Thu, 9 Jan 2020 22:12:04 +0800 Subject: [PATCH 1/9] SPARK-26218: [Follow up] throw exception on overflow for integers revert code style code style save --- .../scala/org/apache/spark/sql/types/numerics.scala | 4 ++-- .../scala/org/apache/spark/sql/SQLQuerySuite.scala | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala index 1ac85360f944f..6288995d63bb9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala @@ -121,8 +121,8 @@ object FloatExactNumeric extends FloatIsFractional { private def overflowException(x: Float, dataType: String) = throw new ArithmeticException(s"Casting $x to $dataType causes overflow") - private val intUpperBound = Int.MaxValue.toFloat - private val intLowerBound = Int.MinValue.toFloat + private val intUpperBound = Int.MaxValue + private val intLowerBound = Int.MinValue private val longUpperBound = Long.MaxValue.toFloat private val longLowerBound = Long.MinValue.toFloat diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index a6dae9a269740..6f9db7ee7a865 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3383,6 +3383,17 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark checkAnswer(df, Row(1)) } } + + test("SPARK-26218: Fix the corner case when casting float to Integer") { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + intercept[ArithmeticException]( + sql("SELECT CAST(CAST(2147483648 as FLOAT) as Integer)").explain() + ) + intercept[ArithmeticException]( + sql("SELECT CAST(CAST(2147483648 as DOUBLE) as Integer)").show() + ) + } + } } case class Foo(bar: Option[String]) From 43866bc772bb2daa07b053cfc967d48502def1a4 Mon Sep 17 00:00:00 2001 From: turbofei Date: Thu, 9 Jan 2020 23:19:46 +0800 Subject: [PATCH 2/9] address comments --- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 6f9db7ee7a865..11f9724e587f2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3387,10 +3387,10 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark test("SPARK-26218: Fix the corner case when casting float to Integer") { withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { intercept[ArithmeticException]( - sql("SELECT CAST(CAST(2147483648 as FLOAT) as Integer)").explain() + sql("SELECT CAST(CAST(2147483648 as FLOAT) as Integer)").collect() ) intercept[ArithmeticException]( - sql("SELECT CAST(CAST(2147483648 as DOUBLE) as Integer)").show() + sql("SELECT CAST(CAST(2147483648 as DOUBLE) as Integer)").collect() ) } } From e3f91919687f99a746179033bdd18e07c6d9035c Mon Sep 17 00:00:00 2001 From: turbofei Date: Fri, 10 Jan 2020 00:07:06 +0800 Subject: [PATCH 3/9] fix style --- .../scala/org/apache/spark/sql/types/numerics.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala index 6288995d63bb9..b5226213effc4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala @@ -123,8 +123,8 @@ object FloatExactNumeric extends FloatIsFractional { private val intUpperBound = Int.MaxValue private val intLowerBound = Int.MinValue - private val longUpperBound = Long.MaxValue.toFloat - private val longLowerBound = Long.MinValue.toFloat + private val longUpperBound = Long.MaxValue + private val longLowerBound = Long.MinValue override def toInt(x: Float): Int = { // When casting floating values to integral types, Spark uses the method `Numeric.toInt` @@ -155,10 +155,10 @@ object DoubleExactNumeric extends DoubleIsFractional { private def overflowException(x: Double, dataType: String) = throw new ArithmeticException(s"Casting $x to $dataType causes overflow") - private val intUpperBound = Int.MaxValue.toDouble - private val intLowerBound = Int.MinValue.toDouble - private val longUpperBound = Long.MaxValue.toDouble - private val longLowerBound = Long.MinValue.toDouble + private val intUpperBound = Int.MaxValue + private val intLowerBound = Int.MinValue + private val longUpperBound = Long.MaxValue + private val longLowerBound = Long.MinValue override def toInt(x: Double): Int = { if (Math.floor(x) <= intUpperBound && Math.ceil(x) >= intLowerBound) { From 4a78aba508cba3e06a4bd9339cc4f245c1d1cd23 Mon Sep 17 00:00:00 2001 From: turbofei Date: Wed, 5 Feb 2020 01:04:44 +0800 Subject: [PATCH 4/9] fix ut --- .../test/resources/sql-tests/inputs/postgreSQL/float4.sql | 2 +- .../resources/sql-tests/results/postgreSQL/float4.sql.out | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql index 2989569e219ff..08775e6b05f30 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql @@ -106,7 +106,7 @@ SELECT smallint(float('32767.6')); SELECT smallint(float('-32768.4')); SELECT smallint(float('-32768.6')); SELECT int(float('2147483520')); -SELECT int(float('2147483647')); +SELECT int(float('2147483547')); SELECT int(float('-2147483648.5')); SELECT int(float('-2147483900')); SELECT bigint(float('9223369837831520256')); diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out index ba913789d5623..46b12b5f0905a 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out @@ -320,11 +320,11 @@ struct -- !query -SELECT int(float('2147483647')) +SELECT int(float('2147483547')) -- !query schema -struct +struct -- !query output -2147483647 +2147483547 -- !query From 0534eb6a0b40e6c5de774bc6e5460de3daa1801c Mon Sep 17 00:00:00 2001 From: turbofei Date: Wed, 5 Feb 2020 03:10:19 +0800 Subject: [PATCH 5/9] fix ut --- .../test/resources/sql-tests/inputs/postgreSQL/float4.sql | 1 - .../resources/sql-tests/results/postgreSQL/float4.sql.out | 8 -------- 2 files changed, 9 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql index 08775e6b05f30..0a052de316717 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql @@ -106,7 +106,6 @@ SELECT smallint(float('32767.6')); SELECT smallint(float('-32768.4')); SELECT smallint(float('-32768.6')); SELECT int(float('2147483520')); -SELECT int(float('2147483547')); SELECT int(float('-2147483648.5')); SELECT int(float('-2147483900')); SELECT bigint(float('9223369837831520256')); diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out index 46b12b5f0905a..f036074c3c074 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out @@ -319,14 +319,6 @@ struct 2147483520 --- !query -SELECT int(float('2147483547')) --- !query schema -struct --- !query output -2147483547 - - -- !query SELECT int(float('-2147483648.5')) -- !query schema From ccad7e64f15eb578a310af4c2a9b202c9d57abbf Mon Sep 17 00:00:00 2001 From: turbofei Date: Wed, 5 Feb 2020 14:04:54 +0800 Subject: [PATCH 6/9] ut --- .../test/resources/sql-tests/inputs/postgreSQL/float4.sql | 1 + .../resources/sql-tests/results/postgreSQL/float4.sql.out | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql index 0a052de316717..ff40b36663661 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql @@ -106,6 +106,7 @@ SELECT smallint(float('32767.6')); SELECT smallint(float('-32768.4')); SELECT smallint(float('-32768.6')); SELECT int(float('2147483520')); +SELECT int(float('2147483392')); SELECT int(float('-2147483648.5')); SELECT int(float('-2147483900')); SELECT bigint(float('9223369837831520256')); diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out index f036074c3c074..e57055cd6a4a1 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out @@ -319,6 +319,14 @@ struct 2147483520 +-- !query +SELECT int(float('2147483392')) +-- !query schema +struct +-- !query output +2147483392 + + -- !query SELECT int(float('-2147483648.5')) -- !query schema From 0ac6e4708cfbb680ec6ca9adcf7131d4d37945a4 Mon Sep 17 00:00:00 2001 From: turbofei Date: Wed, 5 Feb 2020 15:45:59 +0800 Subject: [PATCH 7/9] fix ut --- .../test/resources/sql-tests/inputs/postgreSQL/float4.sql | 2 +- .../resources/sql-tests/results/postgreSQL/float4.sql.out | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql index ff40b36663661..2989569e219ff 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/float4.sql @@ -106,7 +106,7 @@ SELECT smallint(float('32767.6')); SELECT smallint(float('-32768.4')); SELECT smallint(float('-32768.6')); SELECT int(float('2147483520')); -SELECT int(float('2147483392')); +SELECT int(float('2147483647')); SELECT int(float('-2147483648.5')); SELECT int(float('-2147483900')); SELECT bigint(float('9223369837831520256')); diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out index e57055cd6a4a1..fe8375c5eab8f 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out @@ -320,11 +320,12 @@ struct -- !query -SELECT int(float('2147483392')) +SELECT int(float('2147483647')) -- !query schema -struct +struct<> -- !query output -2147483392 +java.lang.ArithmeticException +Casting 2.14748365E9 to int causes overflow -- !query From 356284d5f743de5dad0fea7c9a918d254a5dfa74 Mon Sep 17 00:00:00 2001 From: turbofei Date: Wed, 5 Feb 2020 16:12:02 +0800 Subject: [PATCH 8/9] trigger jenkins --- .../main/scala/org/apache/spark/internal/config/package.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index f91f31be2f1ad..2d0a4045534da 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -876,6 +876,7 @@ package object config { .timeConf(TimeUnit.MILLISECONDS) .createWithDefault(5000) + private[spark] val SHUFFLE_REGISTRATION_MAX_ATTEMPTS = ConfigBuilder("spark.shuffle.registration.maxAttempts") .doc("When we fail to register to the external shuffle service, we will " + From 4d47a49c8aeca5489c3070fb07c15a707254ffb6 Mon Sep 17 00:00:00 2001 From: turbofei Date: Wed, 5 Feb 2020 16:12:15 +0800 Subject: [PATCH 9/9] trigger --- .../main/scala/org/apache/spark/internal/config/package.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 2d0a4045534da..f91f31be2f1ad 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -876,7 +876,6 @@ package object config { .timeConf(TimeUnit.MILLISECONDS) .createWithDefault(5000) - private[spark] val SHUFFLE_REGISTRATION_MAX_ATTEMPTS = ConfigBuilder("spark.shuffle.registration.maxAttempts") .doc("When we fail to register to the external shuffle service, we will " +