From c71a03e2dab0eee8b56107ac552e9b7a2e6ce41a Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Fri, 30 Sep 2016 14:53:30 -0700 Subject: [PATCH 1/6] clean --- .../org/apache/spark/sql/execution/debug/package.scala | 9 --------- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 10 ---------- 2 files changed, 19 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala index d321f4cd76877..dd9d83767e221 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala @@ -69,15 +69,6 @@ package object debug { output } - /** - * Augments [[SparkSession]] with debug methods. - */ - implicit class DebugSQLContext(sparkSession: SparkSession) { - def debug(): Unit = { - sparkSession.conf.set(SQLConf.DATAFRAME_EAGER_ANALYSIS.key, false) - } - } - /** * Augments [[Dataset]]s with debug methods. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index e67140fefef9a..0a72f91aed3f7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -393,14 +393,6 @@ object SQLConf { .intConf .createWithDefault(32) - // Whether to perform eager analysis when constructing a dataframe. - // Set to false when debugging requires the ability to look at invalid query plans. - val DATAFRAME_EAGER_ANALYSIS = SQLConfigBuilder("spark.sql.eagerAnalysis") - .internal() - .doc("When true, eagerly applies query analysis on DataFrame operations.") - .booleanConf - .createWithDefault(true) - // Whether to automatically resolve ambiguity in join conditions for self-joins. // See SPARK-6231. val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY = @@ -733,8 +725,6 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { // See the comments of SCHEMA_STRING_LENGTH_THRESHOLD above for more information. def schemaStringLengthThreshold: Int = getConf(SCHEMA_STRING_LENGTH_THRESHOLD) - def dataFrameEagerAnalysis: Boolean = getConf(DATAFRAME_EAGER_ANALYSIS) - def dataFrameSelfJoinAutoResolveAmbiguity: Boolean = getConf(DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY) From ca4521770274168cf07c83ad62fdfefdaa36c14a Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Fri, 30 Sep 2016 21:58:06 -0700 Subject: [PATCH 2/6] fix msg --- .../main/scala/org/apache/spark/sql/AnalysisException.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 6911843999392..6ca428239f5cc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -45,6 +45,7 @@ class AnalysisException protected[sql] ( override def getMessage: String = { val lineAnnotation = line.map(l => s" line $l").getOrElse("") val positionAnnotation = startPosition.map(p => s" pos $p").getOrElse("") - s"$message;$lineAnnotation$positionAnnotation" + val planAnnotation = plan.map(p => s";\n$p").getOrElse("") + s"$message;$lineAnnotation$positionAnnotation$planAnnotation" } } From 8fb4f40caf50fe40cc1566f476d210ef6c840755 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sat, 1 Oct 2016 09:09:07 -0700 Subject: [PATCH 3/6] fix msg --- .../main/scala/org/apache/spark/sql/AnalysisException.scala | 6 ++++++ .../test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala | 3 +++ 2 files changed, 9 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 6ca428239f5cc..2dca314e16f9e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -48,4 +48,10 @@ class AnalysisException protected[sql] ( val planAnnotation = plan.map(p => s";\n$p").getOrElse("") s"$message;$lineAnnotation$positionAnnotation$planAnnotation" } + + def getSimpleMessage: String = { + val lineAnnotation = line.map(l => s" line $l").getOrElse("") + val positionAnnotation = startPosition.map(p => s" pos $p").getOrElse("") + s"$message;$lineAnnotation$positionAnnotation" + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 55d5a56f1040a..02841d7bb03ff 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -220,6 +220,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted) } catch { + case a: AnalysisException if a.plan.nonEmpty => + // Do not output the logical plan tree which contains expression IDs. + (StructType(Seq.empty), Seq(a.getClass.getName, a.getSimpleMessage)) case NonFatal(e) => // If there is an exception, put the exception class followed by the message. (StructType(Seq.empty), Seq(e.getClass.getName, e.getMessage)) From ab893a677c4864191e92532e2c446e39704937a8 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sat, 1 Oct 2016 09:16:31 -0700 Subject: [PATCH 4/6] clean --- .../main/scala/org/apache/spark/sql/AnalysisException.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 2dca314e16f9e..4fa2af4120fd2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -43,10 +43,8 @@ class AnalysisException protected[sql] ( } override def getMessage: String = { - val lineAnnotation = line.map(l => s" line $l").getOrElse("") - val positionAnnotation = startPosition.map(p => s" pos $p").getOrElse("") val planAnnotation = plan.map(p => s";\n$p").getOrElse("") - s"$message;$lineAnnotation$positionAnnotation$planAnnotation" + getSimpleMessage + planAnnotation } def getSimpleMessage: String = { From 2c5de3cc846b921c51d7550917a6f89252085619 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 10 Oct 2016 10:18:09 -0700 Subject: [PATCH 5/6] add comments --- .../src/main/scala/org/apache/spark/sql/AnalysisException.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala index 4fa2af4120fd2..d3ee8f7a77445 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala @@ -47,6 +47,8 @@ class AnalysisException protected[sql] ( getSimpleMessage + planAnnotation } + // Outputs an exception without the logical plan. + // For testing only def getSimpleMessage: String = { val lineAnnotation = line.map(l => s" line $l").getOrElse("") val positionAnnotation = startPosition.map(p => s" pos $p").getOrElse("") From f082643130e1f39918e06dbe8a39a9a4f49739f5 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 17 Oct 2016 00:01:16 -0700 Subject: [PATCH 6/6] fix --- .../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 29dc2fa912291..8afd39d657865 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -740,10 +740,6 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { def bucketingEnabled: Boolean = getConf(SQLConf.BUCKETING_ENABLED) - // Do not use a value larger than 4000 as the default value of this property. - // See the comments of SCHEMA_STRING_LENGTH_THRESHOLD above for more information. - def schemaStringLengthThreshold: Int = getConf(SCHEMA_STRING_LENGTH_THRESHOLD) - def dataFrameSelfJoinAutoResolveAmbiguity: Boolean = getConf(DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY)