From 64a44998e4ae412645ed38d368639555d3fe1f66 Mon Sep 17 00:00:00 2001 From: Jason Moore Date: Tue, 5 Apr 2016 16:26:11 +1000 Subject: [PATCH 1/4] [SPARK-14357] [CORE] Properly handle the root cause being a commit denied exception --- .../org/apache/spark/executor/Executor.scala | 2 +- .../org/apache/spark/util/CausedBy.scala | 38 +++++++++++ .../org/apache/spark/util/CausedBySuite.scala | 67 +++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 core/src/main/scala/org/apache/spark/util/CausedBy.scala create mode 100644 core/src/test/scala/org/apache/spark/util/CausedBySuite.scala diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 09c57335650c0..afa4d6093a7f2 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -321,7 +321,7 @@ private[spark] class Executor( logInfo(s"Executor killed $taskName (TID $taskId)") execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(TaskKilled)) - case cDE: CommitDeniedException => + case CausedBy(cDE: CommitDeniedException) => val reason = cDE.toTaskEndReason execBackend.statusUpdate(taskId, TaskState.FAILED, ser.serialize(reason)) diff --git a/core/src/main/scala/org/apache/spark/util/CausedBy.scala b/core/src/main/scala/org/apache/spark/util/CausedBy.scala new file mode 100644 index 0000000000000..db6f976b777a9 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/util/CausedBy.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +/** + * Extractor Object for pulling out the root cause of an error. + * If the error contains no cause, it will return the error itself. + * + * Usage: + * try + * { + * ... + * } + * catch { + * case CausedBy(ex: CommitDeniedException) => ... + * } + */ +private[spark] object CausedBy { + + def unapply(e: Throwable): Option[Throwable] = { + Option(e.getCause).flatMap(cause => unapply(cause)).orElse(Some(e)) + } +} diff --git a/core/src/test/scala/org/apache/spark/util/CausedBySuite.scala b/core/src/test/scala/org/apache/spark/util/CausedBySuite.scala new file mode 100644 index 0000000000000..01eae0254fb53 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/util/CausedBySuite.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +import org.apache.spark.SparkFunSuite + +class CausedBySuite extends SparkFunSuite { + + test("For an error without a cause, should return the error") { + + // Arrange + val error = new Exception + + // Act + val causedBy = error match { + case CausedBy(e) => e + } + + // Assert + assert(causedBy === error) + } + + test("For an error with a cause, should return the cause of the error") { + + // Arrange + val cause = new Exception + val error = new Exception(cause) + + // Act + val causedBy = error match { + case CausedBy(e) => e + } + + assert(causedBy === cause) + } + + test("For an error with a cause that itself has a cause, return the root cause") { + + // Arrange + val causeOfCause = new Exception + val cause = new Exception(causeOfCause) + val error = new Exception(cause) + + // Act + val causedBy = error match { + case CausedBy(e) => e + } + + // Assert + assert(causedBy === causeOfCause) + } +} From d1fde47d5b65d4c5a9c661425e958f9b47b267b5 Mon Sep 17 00:00:00 2001 From: Jason Moore Date: Fri, 8 Apr 2016 10:57:03 +1000 Subject: [PATCH 2/4] [SPARK-14357] [CORE] Removed some superfluous comments --- .../scala/org/apache/spark/util/CausedBySuite.scala | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/util/CausedBySuite.scala b/core/src/test/scala/org/apache/spark/util/CausedBySuite.scala index 01eae0254fb53..4a80e3f1f452d 100644 --- a/core/src/test/scala/org/apache/spark/util/CausedBySuite.scala +++ b/core/src/test/scala/org/apache/spark/util/CausedBySuite.scala @@ -22,26 +22,19 @@ import org.apache.spark.SparkFunSuite class CausedBySuite extends SparkFunSuite { test("For an error without a cause, should return the error") { - - // Arrange val error = new Exception - // Act val causedBy = error match { case CausedBy(e) => e } - // Assert assert(causedBy === error) } test("For an error with a cause, should return the cause of the error") { - - // Arrange val cause = new Exception val error = new Exception(cause) - // Act val causedBy = error match { case CausedBy(e) => e } @@ -50,18 +43,14 @@ class CausedBySuite extends SparkFunSuite { } test("For an error with a cause that itself has a cause, return the root cause") { - - // Arrange val causeOfCause = new Exception val cause = new Exception(causeOfCause) val error = new Exception(cause) - // Act val causedBy = error match { case CausedBy(e) => e } - // Assert assert(causedBy === causeOfCause) } } From afa7453438aa6640120b71a77a23e516b1bf961e Mon Sep 17 00:00:00 2001 From: Jason Moore Date: Fri, 8 Apr 2016 10:59:19 +1000 Subject: [PATCH 3/4] [SPARK-14357] [CORE] Make usage comment better reflect style guide --- core/src/main/scala/org/apache/spark/util/CausedBy.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/CausedBy.scala b/core/src/main/scala/org/apache/spark/util/CausedBy.scala index db6f976b777a9..0678d0e8147ef 100644 --- a/core/src/main/scala/org/apache/spark/util/CausedBy.scala +++ b/core/src/main/scala/org/apache/spark/util/CausedBy.scala @@ -22,8 +22,7 @@ package org.apache.spark.util * If the error contains no cause, it will return the error itself. * * Usage: - * try - * { + * try { * ... * } * catch { From d5ebd5585fe7a259cdafb6bb4bd427d567ad93cc Mon Sep 17 00:00:00 2001 From: Jason Moore Date: Fri, 8 Apr 2016 11:15:18 +1000 Subject: [PATCH 4/4] [SPARK-14357] [CORE] Make usage comment better reflect style guide (again) --- core/src/main/scala/org/apache/spark/util/CausedBy.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/CausedBy.scala b/core/src/main/scala/org/apache/spark/util/CausedBy.scala index 0678d0e8147ef..73df446d981cb 100644 --- a/core/src/main/scala/org/apache/spark/util/CausedBy.scala +++ b/core/src/main/scala/org/apache/spark/util/CausedBy.scala @@ -24,8 +24,7 @@ package org.apache.spark.util * Usage: * try { * ... - * } - * catch { + * } catch { * case CausedBy(ex: CommitDeniedException) => ... * } */