From 60b76e0ec02619ca288d672398584c73903dfcd7 Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Tue, 21 Jun 2016 20:22:33 -0700 Subject: [PATCH 1/7] [TRIVIAL] [CORE] [ScriptTransform] move printing of stderr buffer before closing the outstream --- .../apache/spark/sql/hive/execution/ScriptTransformation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index 84990d369786b..fabd51a8c34c3 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -322,10 +322,10 @@ private class ScriptTransformationWriterThread( throw e } finally { try { - outputStream.close() if (proc.waitFor() != 0) { logError(stderrBuffer.toString) // log the stderr circular buffer } + outputStream.close() } catch { case NonFatal(exceptionFromFinallyBlock) => if (!threwException) { From 2b7b9258896643e2fda7434f88eca7731737a42a Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Thu, 30 Jun 2016 23:14:37 -0700 Subject: [PATCH 2/7] fix unit test failures --- .../apache/spark/sql/hive/execution/ScriptTransformation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index fabd51a8c34c3..1efc998e13b13 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -313,6 +313,7 @@ private class ScriptTransformationWriterThread( } } threwException = false + outputStream.close() } catch { case NonFatal(e) => // An error occurred while writing input, so kill the child process. According to the @@ -325,7 +326,6 @@ private class ScriptTransformationWriterThread( if (proc.waitFor() != 0) { logError(stderrBuffer.toString) // log the stderr circular buffer } - outputStream.close() } catch { case NonFatal(exceptionFromFinallyBlock) => if (!threwException) { From e2277ff9685b82be4d2b38df7228cd37bf62ba4f Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Tue, 5 Jul 2016 07:45:23 -0700 Subject: [PATCH 3/7] - Catch Runtime --- .../spark/sql/hive/execution/ScriptTransformation.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index 1efc998e13b13..ab772905ec4d7 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -312,10 +312,10 @@ private class ScriptTransformationWriterThread( } } } + Utils.tryLogNonFatalError(outputStream.close()) threwException = false - outputStream.close() } catch { - case NonFatal(e) => + case e @ Throwable => // An error occurred while writing input, so kill the child process. According to the // Javadoc this call will not throw an exception: _exception = e From 2fe6e856e2949f624e82f492d2f1f135eae0f8c6 Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Tue, 5 Jul 2016 09:11:48 -0700 Subject: [PATCH 4/7] move close inside try --- .../apache/spark/sql/hive/execution/ScriptTransformation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index ab772905ec4d7..826c5d1a4ab52 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -312,7 +312,6 @@ private class ScriptTransformationWriterThread( } } } - Utils.tryLogNonFatalError(outputStream.close()) threwException = false } catch { case e @ Throwable => @@ -323,6 +322,7 @@ private class ScriptTransformationWriterThread( throw e } finally { try { + Utils.tryLogNonFatalError(outputStream.close()) if (proc.waitFor() != 0) { logError(stderrBuffer.toString) // log the stderr circular buffer } From c39c43fc7a9e4fd965a2a44aaf0ec724b507e112 Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Tue, 5 Jul 2016 09:16:39 -0700 Subject: [PATCH 5/7] Use Nonfatal --- .../apache/spark/sql/hive/execution/ScriptTransformation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index 826c5d1a4ab52..5824043d11fb1 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -314,7 +314,7 @@ private class ScriptTransformationWriterThread( } threwException = false } catch { - case e @ Throwable => + case NonFatal(e) => // An error occurred while writing input, so kill the child process. According to the // Javadoc this call will not throw an exception: _exception = e From 2cb1669671633c9fac4b9b44ed63702c28dd6b11 Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Tue, 5 Jul 2016 09:18:03 -0700 Subject: [PATCH 6/7] Change Throwable --- .../apache/spark/sql/hive/execution/ScriptTransformation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index 5824043d11fb1..a01fb0aa50839 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -314,7 +314,7 @@ private class ScriptTransformationWriterThread( } threwException = false } catch { - case NonFatal(e) => + case t: Throwable => // An error occurred while writing input, so kill the child process. According to the // Javadoc this call will not throw an exception: _exception = e From 7b58a040e3d1a0eb592c4a19592e2d6ec7179fd7 Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Tue, 5 Jul 2016 09:37:15 -0700 Subject: [PATCH 7/7] rename e => t --- .../spark/sql/hive/execution/ScriptTransformation.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index a01fb0aa50839..d063dd6b7f599 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -317,9 +317,9 @@ private class ScriptTransformationWriterThread( case t: Throwable => // An error occurred while writing input, so kill the child process. According to the // Javadoc this call will not throw an exception: - _exception = e + _exception = t proc.destroy() - throw e + throw t } finally { try { Utils.tryLogNonFatalError(outputStream.close())