From 3b3f224d6ac2dc3d3a0c21ed14502329af3cbae8 Mon Sep 17 00:00:00 2001 From: baibing Date: Fri, 27 Jul 2018 15:49:50 +0800 Subject: [PATCH 1/2] [SPARK-24931][CORE]CoarseGrainedExecutorBackend send wrong 'Reason' when executor exits which leading to job failed. --- .../apache/spark/executor/CoarseGrainedExecutorBackend.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index 48d3630abd1f9..170b9e95633de 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -32,7 +32,7 @@ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.deploy.worker.WorkerWatcher import org.apache.spark.internal.Logging import org.apache.spark.rpc._ -import org.apache.spark.scheduler.{ExecutorLossReason, TaskDescription} +import org.apache.spark.scheduler.{ExecutorExited, TaskDescription} import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._ import org.apache.spark.serializer.SerializerInstance import org.apache.spark.util.{ThreadUtils, Utils} @@ -165,7 +165,7 @@ private[spark] class CoarseGrainedExecutorBackend( } if (notifyDriver && driver.nonEmpty) { - driver.get.send(RemoveExecutor(executorId, new ExecutorLossReason(reason))) + driver.get.send(RemoveExecutor(executorId, new ExecutorExited(code, false, reason))) } System.exit(code) From bb8f029c6626cf3834315abe509e7ace73bb7e34 Mon Sep 17 00:00:00 2001 From: baibing Date: Mon, 13 Aug 2018 16:04:09 +0800 Subject: [PATCH 2/2] =?UTF-8?q?[SPARK-25098][SQL]=E2=80=98Cast=E2=80=99=20?= =?UTF-8?q?will=20return=20NULL=20when=20input=20string=20starts/ends=20wi?= =?UTF-8?q?th=20special=20character(s)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/spark/sql/catalyst/expressions/Cast.scala | 7 +++++-- .../apache/spark/sql/catalyst/expressions/CastSuite.scala | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index ba4d1314bab2b..adb6e9d55c0b2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -350,7 +350,8 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String // TimestampConverter private[this] def castToTimestamp(from: DataType): Any => Any = from match { case StringType => - buildCast[UTF8String](_, utfs => DateTimeUtils.stringToTimestamp(utfs, timeZone).orNull) + buildCast[UTF8String](_, utfs => DateTimeUtils.stringToTimestamp( + UTF8String.fromString(utfs.toString.trim), timeZone).orNull) case BooleanType => buildCast[Boolean](_, b => if (b) 1L else 0) case LongType => @@ -393,7 +394,9 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String // DateConverter private[this] def castToDate(from: DataType): Any => Any = from match { case StringType => - buildCast[UTF8String](_, s => DateTimeUtils.stringToDate(s).orNull) + buildCast[UTF8String](_, s => DateTimeUtils.stringToDate( + UTF8String.fromString(s.toString.trim) + ).orNull) case TimestampType => // throw valid precision more than seconds, according to Hive. // Timestamp.nanos is in 0 to 999,999,999, no more than a second. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index 5b25bdf907c3a..32bfb8a152a3b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -98,6 +98,7 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { c.set(Calendar.MILLISECOND, 0) checkEvaluation(Cast(Literal("2015-03-18"), DateType), new Date(c.getTimeInMillis)) checkEvaluation(Cast(Literal("2015-03-18 "), DateType), new Date(c.getTimeInMillis)) + checkEvaluation(Cast(Literal(" 2015-03-18"), DateType), new Date(c.getTimeInMillis)) checkEvaluation(Cast(Literal("2015-03-18 123142"), DateType), new Date(c.getTimeInMillis)) checkEvaluation(Cast(Literal("2015-03-18T123123"), DateType), new Date(c.getTimeInMillis)) checkEvaluation(Cast(Literal("2015-03-18T"), DateType), new Date(c.getTimeInMillis)) @@ -130,6 +131,7 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { c.set(Calendar.MILLISECOND, 0) checkCastStringToTimestamp("2015-03-18", new Timestamp(c.getTimeInMillis)) checkCastStringToTimestamp("2015-03-18 ", new Timestamp(c.getTimeInMillis)) + checkCastStringToTimestamp(" 2015-03-18", new Timestamp(c.getTimeInMillis)) checkCastStringToTimestamp("2015-03-18T", new Timestamp(c.getTimeInMillis)) c = Calendar.getInstance(tz)