From a633d2b492ebe26eace5eb0dd9ec049a08e3a67f Mon Sep 17 00:00:00 2001 From: chenzhx Date: Mon, 18 Oct 2021 17:48:03 +0800 Subject: [PATCH 1/2] Reduce the scope of synchronized to prevent potential deadlock --- .../sql/connector/catalog/CatalogManager.scala | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala index 0779bf53fe446..49ebf0f9710e4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala @@ -88,12 +88,16 @@ class CatalogManager( private var _currentNamespace: Option[Array[String]] = None - def currentNamespace: Array[String] = synchronized { - _currentNamespace.getOrElse { - if (currentCatalog.name() == SESSION_CATALOG_NAME) { - Array(v1SessionCatalog.getCurrentDatabase) - } else { - currentCatalog.defaultNamespace() + def currentNamespace: Array[String] = { + val defaultNamespace = if (currentCatalog.name() == SESSION_CATALOG_NAME) { + Array(v1SessionCatalog.getCurrentDatabase) + } else { + currentCatalog.defaultNamespace() + } + + this.synchronized { + _currentNamespace.getOrElse { + defaultNamespace } } } From a902f422b75f5604ecbaa93167fc76a3e60c1b0b Mon Sep 17 00:00:00 2001 From: chenzhx Date: Tue, 19 Oct 2021 17:21:59 +0800 Subject: [PATCH 2/2] upgrade spark version r38 --- assembly/pom.xml | 2 +- common/kvstore/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/avro/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10-token-provider/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- hadoop-cloud/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 36 files changed, 36 insertions(+), 36 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index b909935b16c15..2044ad4297d7c 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index 87202b31127c1..cf47a459a6314 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 274e093feb663..145f9a5b53ea3 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index a1cc152b18cc0..dce1bc6acfe22 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 5b3b502e66dac..c09921af2083d 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 43f69394fff28..ef09d717bafde 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 026f5441c955c..51b8b326f76d6 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index c2db299478f58..1ea5f8a0ea8b3 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 2f2284231317e..8df87209da43a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index 306f2a70dc175..0f052e2b6b907 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index 9152656a73100..0e077a927afd9 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index daa4cd42eee35..db9722cbcda27 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index e51d7fec20086..e4d92b5291248 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index b3e3542039b15..c2e7f97babac8 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml index 119235391f2b3..63d6c49a8a5ac 100644 --- a/external/kafka-0-10-token-provider/pom.xml +++ b/external/kafka-0-10-token-provider/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index dad67c0dffe9c..f6576f91b0228 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 8d21dabc34eeb..e0717887983bf 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index c12616568da10..15c08ca74a12b 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index 8ed1ad8e92b6d..6d6e5ef75e4d1 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index a28af61b9855b..45c31a9f5e08d 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index ddffce5f61873..a04dc0956ad90 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 623f4af338122..e169bb08fa431 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index cf2f3c0cfe296..e97e5717471f4 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index fdef53639e5e6..e116dc0fa1ddb 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/pom.xml b/pom.xml index 53cc3feed5250..c32825fa6912f 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index 1bf014d1ce824..3f67b03168731 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 0c015c53fbba9..1215ddc50dc3c 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 600da6506fdd7..4360384343c4a 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index da6990756755b..0aa7451ab1a53 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 130fff5b92d2b..a07809cec35c0 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 2b24634a4d28a..b754629cd7eb5 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 656b0b5350d4e..16982a13edc57 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 382f1e33b0af5..237f1ded0963c 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index e6d05e383920e..bb84214ea8789 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index c71c3d0fde102..bd9b1f9594b62 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 8fcf9e6fab51c..79ae2cc7b277c 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r37 + 3.1.1-kylin-4.x-r38 ../pom.xml